diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9c7cbddd..c8f84986 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -21,6 +21,7 @@ - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Release preparation ## Checklist: diff --git a/.github/workflows/docker-compose.yaml b/.github/workflows/docker-compose.yaml new file mode 100644 index 00000000..ab3608ab --- /dev/null +++ b/.github/workflows/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + xdmod10.5: + container_name: xdmod10.5 + image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3 + tty: true + ports: + - "8081:443" + xdmod-main-dev: + container_name: xdmod-main-dev + image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3 + tty: true + ports: + - "8082:443" diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3f9ee2bc..2d5da2c4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + python-version: [ + "3.12", + "3.11", + "3.10", + "3.9", + "3.8", + ] steps: - uses: actions/checkout@v3 @@ -27,15 +33,69 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest pandas + python -m pip install flake8 flake8-commas flake8-quotes pandas python-dotenv pytest coverage if [ -f requirements.txt ]; then pip install -r requirements.txt; fi python -m pip install -e . - name: Lint with flake8 run: | # stop the build if there are linter errors (ignore complexity metrics) - flake8 . --count --max-line-length=127 --max-complexity=1000 --show-source --statistics --exclude __init__.py - # output complexity metrics - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude __init__.py - - name: Test with pytest + flake8 . --max-complexity=10 --show-source --exclude __init__.py + - name: Spin up containers for testing XDMoD webservers + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: .github/workflows/docker-compose.yaml + - name: Install yq run: | - pytest tests/unit + sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq + - name: Test against each supported XDMoD portal version + run: | + # Create a file with 10,000 users so that we can test filters that have more than 10,000 values + for i in {1..10000}; do + # sets the unix at the end of quarter 2 + unix=$((1498866840+$i*2)) + # each user is added in the file later to be injested + echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log + done + #run test + coverage erase + declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml) + for version in $portal_versions; do + echo "starting up $version container" + # Generate OpenSSL Key + docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key' + # Generate Certificate + docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost" -addext 'subjectAltName=DNS:localhost' + # Copy Certificate from container + docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt + if [ "$version" = "xdmod-main-dev" ]; then + # Install the latest development version of xdmod, + # these steps come from xdmod CircleCI configuration file + docker exec xdmod-main-dev bash -c "dnf module -y reset php && dnf module -y enable php:7.4" + docker exec xdmod-main-dev bash -c "dnf install -y php libzip-devel php-pear php-devel" + docker exec xdmod-main-dev bash -c "yes '' | pecl install mongodb || true" + docker exec xdmod-main-dev bash -c "git clone --depth=1 https://github.com/ubccr/xdmod.git /root/xdmod" + docker exec -w /root/xdmod xdmod-main-dev bash -c "composer install" + docker exec -w /root/xdmod xdmod-main-dev bash -c "~/bin/buildrpm xdmod" + docker exec -w /root/xdmod xdmod-main-dev bash -c "XDMOD_TEST_MODE=upgrade ./tests/ci/bootstrap.sh" + docker exec -w /root/xdmod xdmod-main-dev bash -c "./tests/ci/validate.sh" + else + # Start up web server in container + docker exec $version /root/bin/services start + fi + # Prepare to use certificate for curl and python requests + export CURL_CA_BUNDLE="$version.crt" + ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml) + port=$(echo $ports | cut -d ':' -f 1) + export XDMOD_HOST="https://localhost:$port" + token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token') + echo -n 'XDMOD_API_TOKEN=' > ~/.xdmod-data-token + curl -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token + docker cp 10000users.log $version:10000users.log + docker exec $version xdmod-shredder -r frearson -f slurm -i 10000users.log + date=$(date --utc +%Y-%m-%d) + docker exec $version xdmod-ingestor --ingest + docker exec $version xdmod-ingestor --aggregate=job --last-modified-start-date $date + export XDMOD_VERSION="$version" + coverage run --branch --append -m pytest -vvs -o log_cli=true tests/ + done + coverage report -m --fail-under=100 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index aada8df2..59f2daf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,23 @@ # xdmod-data Changelog ## Main development branch +- Implement 100% test coverage ([\#27](https://github.com/ubccr/xdmod-data/pull/27)). +- Update Flake8 rules ([\#28](https://github.com/ubccr/xdmod-data/pull/28)). + +## v1.0.1 (2024-06-XX) +This release has bug fixes, performance improvements, and updates for compatibility, tests, and documentation. +- Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)). +- Create `PULL_REQUEST_TEMPLATE.md` ([\#13](https://github.com/ubccr/xdmod-data/pull/13)). +- Add types of changes to pull request template ([\#15](https://github.com/ubccr/xdmod-data/pull/15)). +- Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)). - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)). - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)). -- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)). +- Add citation for the DAF paper ([\#23](https://github.com/ubccr/xdmod-data/pull/23)). +- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-data/pull/22)). - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)). +- Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)). +- Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)). +- Specify minimum version numbers for dependencies ([\#25](https://github.com/ubccr/xdmod-data/pull/25)). ## v1.0.0 (2023-07-21) - Initial release. diff --git a/README.md b/README.md index 0c9263a9..da1d4659 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # xdmod-data -As part of the [XDMoD](https://open.xdmod.org) Data Analytics Framework, this Python package provides API access to the data warehouse of an instance of XDMoD version ≥10.5. +As part of the Data Analytics Framework for [XDMoD](https://open.xdmod.org), this Python package provides API access to the data warehouse of instances of Open XDMoD. The package can be installed from PyPI via `pip install xdmod-data`. @@ -7,6 +7,14 @@ It has dependencies on [NumPy](https://pypi.org/project/numpy/), [Pandas](https: Example usage is documented through Jupyter notebooks in the [xdmod-notebooks](https://github.com/ubccr/xdmod-notebooks) repository. +## Compatibility with Open XDMoD +Specific versions of this `xdmod-data` API are compatible with specific versions of Open XDMoD as indicated in the table below. + +| `xdmod-data` version | Open XDMoD versions | +| -------------------- | ------------------- | +| 1.0.1 | 11.0.x, 10.5.x | +| 1.0.0 | 10.5.x | + ## API Token Access Use of the Data Analytics Framework requires an API token. To obtain an API token, follow the steps below to obtain an API token from the XDMoD portal. @@ -55,7 +63,13 @@ For support, please see [this page](https://open.xdmod.org/support.html). If you ## License `xdmod-data` is released under the GNU Lesser General Public License ("LGPL") Version 3.0. See the [LICENSE](LICENSE) file for details. -## Reference +## References + +When referencing the Data Analytics Framework for XDMoD, please cite the following publication: + +> Weeden, A., White, J.P., DeLeon, R.L., Rathsam, R., Simakov, N.A., Saeli, C., and Furlani, T.R. The Data Analytics Framework for XDMoD. _SN COMPUT. SCI._ 5, 462 (2024). https://doi.org/10.1007/s42979-024-02789-2 + When referencing XDMoD, please cite the following publication: -> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:10.1109/MCSE.2015.68 +> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:[10.1109/MCSE.2015.68](https://doi.org/10.1109/MCSE.2015.68) + diff --git a/docs/developing.md b/docs/developing.md index 23045149..816746f7 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -1,53 +1,114 @@ -# Testing the code +# Instructions for developers + +## Testing the code 1. Start up a virtual environment, e.g.: + ``` + python3 -m venv ~/xdmod-data-test-env + source ~/xdmod-data-test-env/bin/activate ``` - $ env_dir=~/xdmod-data-test-env - $ python3 -m venv ${env_dir} - $ source ${env_dir}/bin/activate - ``` + Your command prompt should now start with `(xdmod-data-test-env)`. 1. Install your local development copy of `xdmod-data` in editable mode: ``` - (env) $ python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data + python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data ``` -1. Install `python-dotenv` and `pytest`: +1. Install `python-dotenv`, `pytest`, and `coverage`: ``` - (env) $ python3 -m pip install --upgrade python-dotenv pytest + python3 -m pip install --upgrade python-dotenv pytest coverage ``` 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600. -1. With an [https://xdmod.access-ci.org](ACCESS XDMoD) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token). +1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token). 1. Add the following line to the file `~/.xdmod-data-token`, replacing `` with your token. ``` XDMOD_API_TOKEN= ``` 1. Change directories to your local development copy of `xdmod-data`. -1. Run the following command: +1. Run the following command and make sure all the tests pass: ``` - (env) $ pytest -vvs -o log_cli=true tests/ + coverage run -m pytest -vvs -o log_cli=true tests/ ``` - -# Developing a new version -1. Make sure the version number is updated in `xdmod_data/__version__.py`. -1. Create a Pull Request for the new version. - -# Distributing the new version to PyPI -After the Pull Request is approved, follow these steps in the `xdmod-data` root directory: -1. Start up a virtual environment, e.g.: +1. Run the following command and make sure the code is 100% covered by tests: + ``` + coverage report -m ``` - $ env_dir=~/xdmod-data-build-env - $ python3 -m venv ${env_dir} - $ source ${env_dir}/bin/activate +1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`. ``` -1. Make sure the required packages are installed: + python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0 ``` - (env) $ python3 -m pip install --upgrade pip build setuptools twine +1. Run the following command again and make sure all the tests pass (Deprecation warnings in `urllib3` are Ok). ``` -1. Build the built distribution: + pytest -vvs -o log_cli=true tests/ + ``` +## Linting the code +1. Start up a virtual environment, e.g.: + ``` + python3 -m venv ~/xdmod-data-test-env + source ~/xdmod-data-test-env/bin/activate ``` - (env) $ python3 -m build --wheel + Your command prompt should now start with `(xdmod-data-test-env)`. +1. Install Flake8 and additional packages ``` -1. Upload the built distribution to PyPI, e.g., for version 1.0.0-beta1: + python3 -m pip install flake8 flake8-quotes flake8-commas ``` - (env) $ version=1.0.0b1 - (env) $ twine upload dist/xdmod_data-${version}-py3-none-any.whl +1. Run Flake8 ``` - Enter your PyPI username and password. + flake8 --max-complexity=10 --show-source --exclude __init__.py /path/to/your/xdmod-data + ``` +## Releasing a new version +1. Make a new branch of `xdmod-data` and: + + 1. Make sure the version number is updated in `xdmod_data/__version__.py`. + 1. Update the Open XDMoD compatibility matrix in the `README.md`. + 1. Update `CHANGELOG.md` to change "Main development branch" to, e.g., `v1.0.1 (2024-06-XX)`. + 1. Create a Pull Request for the new version. + +1. After the Pull Request is approved (but not merged yet), follow these steps in a cloned copy of the branch: + 1. Start up a virtual environment, e.g.: + ``` + python3 -m venv ~/xdmod-data-build-env + source ~/xdmod-data-build-env/bin/activate + ``` + Your command prompt should now start with `(xdmod-data-build-env)`. + 1. Make sure the required packages are installed: + ``` + python3 -m pip install --upgrade pip build setuptools twine + ``` + 1. Build the built distribution: + ``` + python3 -m build --wheel + ``` + 1. Validate `setup.cfg`, e.g., for version 1.0.0-beta1: + ``` + version=1.0.0b1 + twine check dist/xdmod_data-${version}-py3-none-any.whl + ``` + Make sure you receive `PASSED`. + 1. Upload the built distribution to TestPyPI: + ``` + twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl + ``` + Enter your TestPyPI API token. + 1. Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right. + 1. Upload the built distribution to PyPI: + ``` + twine upload dist/xdmod_data-${version}-py3-none-any.whl + ``` + Enter your PyPI username and password. + 1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release. +1. Update the date of the release in `CHANGELOG.md` and commit it to the Pull Request. +1. Merge the Pull Request. + +1. Go to [create a new release on GitHub](https://github.com/ubccr/xdmod-data/releases/new) and: + 1. Click `Choose a tag`. + 1. Type in a tag similar to `v1.0.0` and choose `Create new tag`. + 1. Make the release title the same as the tag name. + 1. Where it says `Describe this release` paste in the contents of `CHANGELOG.md`. + 1. Where it says `Attach binaries` attach the built distribution that was uploaded to PyPI. + 1. Click `Publish release`. + +## After release + +1. Make a new branch of `xdmod-data` and: + + 1. Make sure the version number is updated in `xdmod_data/__version__.py` to a beta release of the next version, e.g., `1.0.1-beta.1`. + 1. Update `CHANGELOG.md` to add a section at the top called `Main development branch`. + 1. Create a Pull Request for the new version. \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 0333c62a..9839013b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ version = attr: xdmod_data.__version__ author = Aaron Weeden, Joseph P. White maintainer = Aaron Weeden, Joseph P. White description = Python package for XDMoD data access -long_description = file: README.md +long_description = See [the README](https://github.com/ubccr/xdmod-data?tab=readme-ov-file#xdmod-data) for instructions on use. long_description_content_type = text/markdown license_files = LICENSE project_urls = @@ -12,9 +12,9 @@ project_urls = [options] packages = find: -python_requires = >=3 +python_requires = >=3.8 install_requires = - numpy - pandas - plotly - requests + numpy >= 1.23.0 + pandas >= 1.5.0 + plotly >= 5.8.0 + requests >= 2.19.0 diff --git a/tests/integration/test_datawarehouse_integration.py b/tests/integration/test_datawarehouse_integration.py index 9e296fec..6f7ee5df 100644 --- a/tests/integration/test_datawarehouse_integration.py +++ b/tests/integration/test_datawarehouse_integration.py @@ -1,11 +1,12 @@ from dotenv import load_dotenv +import os from os.path import expanduser import pandas from pathlib import Path import pytest from xdmod_data.warehouse import DataWarehouse -VALID_XDMOD_URL = 'https://xdmod.access-ci.org' +VALID_XDMOD_HOST = os.environ['XDMOD_HOST'] TOKEN_PATH = '~/.xdmod-data-token' INVALID_STR = 'asdlkfjsdlkfisdjkfjd' METHOD_PARAMS = { @@ -28,23 +29,23 @@ 'describe_realms': (), 'describe_metrics': ('realm',), 'describe_dimensions': ('realm',), - 'get_filter_values': ('realm', 'dimension',), + 'get_filter_values': ('realm', 'dimension'), 'describe_raw_realms': (), 'describe_raw_fields': ('realm',), } -VALID_DATE = '2020-01-01' +VALID_DATE = '2016-12-25' VALID_DIMENSION = 'Resource' VALID_VALUES = { - 'duration': 'Previous month', + 'duration': 'Yesterday', 'realm': 'Jobs', 'metric': 'CPU Hours: Total', 'dimension': VALID_DIMENSION, - 'filters': {VALID_DIMENSION: 'Expanse'}, + 'filters': {VALID_DIMENSION: 'phillips'}, 'dataset_type': 'timeseries', 'aggregation_unit': 'Auto', 'parameter': 'duration', 'fields': ['Nodes'], - 'show_progress': True, + 'show_progress': False, } KEY_ERROR_TEST_VALUES_AND_MATCHES = { 'duration': (INVALID_STR, 'Invalid value for `duration`'), @@ -53,7 +54,8 @@ 'dimension': (INVALID_STR, r'Dimension .* not found'), 'filter_key': ({INVALID_STR: INVALID_STR}, r'Dimension .* not found'), 'filter_value': ( - {VALID_DIMENSION: INVALID_STR}, r'Filter value .* not found' + {VALID_DIMENSION: INVALID_STR}, + r'Filter value .* not found', ), 'dataset_type': (INVALID_STR, 'Invalid value for `dataset_type`'), 'aggregation_unit': (INVALID_STR, 'Invalid value for `aggregation_unit`'), @@ -92,8 +94,16 @@ method + ':end_date', ] date_malformed_test_params += [ - (method, 'start_date', {'duration': (INVALID_STR, VALID_DATE)}), - (method, 'end_date', {'duration': (VALID_DATE, INVALID_STR)}), + ( + method, + 'start_date', + {'duration': (INVALID_STR, VALID_DATE)}, + ), + ( + method, + 'end_date', + {'duration': (VALID_DATE, INVALID_STR)}, + ), ] value_error_test_methods += [method] if 'filters' in METHOD_PARAMS[method]: @@ -107,14 +117,17 @@ @pytest.fixture(scope='module') -def dw_methods(): - with DataWarehouse(VALID_XDMOD_URL) as dw: +def dw_methods(request): + xdmod_host = VALID_XDMOD_HOST + if hasattr(request, 'param'): + xdmod_host = request.param + with DataWarehouse(xdmod_host) as dw: yield __get_dw_methods(dw) @pytest.fixture(scope='module') def dw_methods_outside_runtime_context(): - dw = DataWarehouse(VALID_XDMOD_URL) + dw = DataWarehouse(VALID_XDMOD_HOST) return __get_dw_methods(dw) @@ -152,16 +165,7 @@ def test_KeyError(dw_methods, method, params, match): @pytest.mark.parametrize( 'method', - [ - 'get_data', - 'get_raw_data', - 'describe_realms', - 'describe_metrics', - 'describe_dimensions', - 'get_filter_values', - 'describe_raw_realms', - 'describe_raw_fields', - ], + list(METHOD_PARAMS.keys()), ) def test_RuntimeError_outside_context( dw_methods_outside_runtime_context, @@ -248,7 +252,7 @@ def __test_DataFrame_return_value( get_data_return_value_test_params = { - 'duration': ('2020-01-01', '2020-01-31'), + 'duration': ('2016-12-22', '2017-01-31'), 'realm': 'Jobs', 'metric': 'Number of Users: Active', 'dimension': 'None', @@ -264,25 +268,25 @@ def __test_DataFrame_return_value( ( {}, 'Metric', - 31, + 11, ), ( - {'filters': {'Service Provider': 'StonyBrook'}}, + {'filters': {'Resource': 'robertson'}}, 'Metric', - 0, + 6, ), ( {'dimension': 'Resource'}, 'Resource', - 31, + 11, ), ( { 'dimension': 'Resource', - 'filters': {'Service Provider': 'StonyBrook'} + 'filters': {'Resource': 'robertson'}, }, 'Resource', - 0, + 6, ), ], ids=( @@ -338,22 +342,22 @@ def test_get_data_timeseries_return_value( 1, ), ( - {'filters': {'Service Provider': 'StonyBrook'}}, + {'filters': {'Resource': 'robertson'}}, None, 1, ), ( {'dimension': 'Resource'}, 'Resource', - 8, + 5, ), ( { 'dimension': 'Resource', - 'filters': {'Service Provider': 'StonyBrook'} + 'filters': {'Resource': 'robertson'}, }, 'Resource', - 0, + 1, ), ], ids=( @@ -439,3 +443,18 @@ def test_case_insensitive(dw_methods, method, param, value1, value2): data1 = __run_method(dw_methods, method, {param: value1}) data2 = __run_method(dw_methods, method, {param: value2}) assert data1.equals(data2) + + +@pytest.mark.parametrize( + 'dw_methods,method', + [ + ( + VALID_XDMOD_HOST + '/', + method, + ) for method in list(METHOD_PARAMS.keys()) + ], + indirect=['dw_methods'], + ids=[method for method in list(METHOD_PARAMS.keys())], +) +def test_trailing_slashes(dw_methods, method): + __run_method(dw_methods, method) diff --git a/tests/regression/data/jobs-dimensions.csv b/tests/regression/data/jobs-dimensions.csv deleted file mode 100644 index 798d3541..00000000 --- a/tests/regression/data/jobs-dimensions.csv +++ /dev/null @@ -1,26 +0,0 @@ -id,label,description -none,None,Summarizes jobs reported to the ACCESS allocations service (excludes non-ACCESS usage of the resource). -allocation,Allocation,A funded project that is allowed to run jobs on resources. -fieldofscience,Field of Science,The field of science indicated on the allocation request pertaining to the running jobs. -gateway,Gateway,A science gateway is a portal set up to aid submiting jobs to resources. -grant_type,Grant Type,A categorization of the projects/allocations. -jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job. -jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited. -jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute. -nsfdirectorate,NSF Directorate,The NSF directorate of the field of science indiciated on the allocation request pertaining to the running jobs. -nodecount,Node Count,A categorization of jobs into discrete groups based on node count. -pi,PI,The principal investigator of a project. -pi_institution,PI Institution,Organizations that have PIs with allocations. -pi_institution_country,PI Institution Country,The country of the institution of the PI of the project associated with compute jobs. -pi_institution_state,PI Institution State,The location of the institution of the PI of the project associated with the compute jobs. -parentscience,Parent Science,The parent of the field of science indiciated on the allocation request pertaining to the running jobs. -queue,Queue,Queue pertains to the low level job queues on each resource. -resource,Resource,A resource is a remote computer that can run jobs. -resource_type,Resource Type,A categorization of resources into by their general capabilities. -provider,Service Provider,A service provider is an institution that hosts resources. -username,System Username,The specific system username of the users who ran jobs. -person,User,"A person who is on a PIs allocation, hence able run jobs on resources." -institution,User Institution,Organizations that have users with allocations. -institution_country,User Institution Country,The name of the country of the institution of the person who ran the compute job. -institution_state,User Institution State,The location of the institution of the person who ran the compute job. -nsfstatus,User NSF Status,Categorization of the users who ran jobs. diff --git a/tests/regression/data/jobs-fieldofscience-filter-values.csv b/tests/regression/data/jobs-fieldofscience-filter-values.csv deleted file mode 100644 index 86ccb36e..00000000 --- a/tests/regression/data/jobs-fieldofscience-filter-values.csv +++ /dev/null @@ -1,82 +0,0 @@ -id,label -189,"Agriculture, Forestry, and Fisheries" -192,Agriculural Biotechnology -236,Analytical and Materials Chemistry -219,Applied Computer Science -215,Applied Mathematics -221,Artificial Intelligence and Intelligent Systems -206,Arts -225,Astronomy and Planetary Sciences -238,Atmospheric Sciences -228,"Atomic, Molecular, and Optical Physics" -184,Basic Medicine -254,Behavioral Sciences Biology -247,Biochemistry and Molecular Biology -246,Biophysics -248,Cell Biology -176,Chemical Engineering -173,Civil Engineering -239,Climate and Global Dynamics -185,Clinical Medicine -218,Computer Science -226,Condensed Matter Physics -250,Developmental Biology -252,Ecology -195,Economics and Business -196,Educational Sciences -174,"Electrical, Electronic, and Information Engineering" -253,Environmental Biology -180,Environmental Biotechnology -179,Environmental Engineering -231,Fluid and Plasma Physics -249,Genetics -242,Geology and Solid Earth Sciences -241,Geophysics and Geochemistry -227,Gravitational Physics -186,Health Sciences -203,History and Archaeology -244,Hydrology and Water Resources -181,Industrial Biotechnology -223,"Informatics, Analytics and Information Science" -211,Infrastructure and Instrumentation -235,Inorganic and Nuclear Chemistry -198,Law -240,Magnetospheric and Upper Atmospheric Physics -177,Materials Engineering -175,Mechanical Engineering -201,Media and communications -187,Medical Biotechnology -178,Medical Engineering -182,Nanotechnology -229,Nuclear Physics -243,Oceanography and Ocean Sciences -234,Organic Chemistry -213,Other -193,Other Agricultural Sciences -255,Other Biological Sciences -237,Other Chemical Sciences -224,Other Computer and Information Sciences -245,Other Earth and Environmental Sciences -183,Other Engineering and Technologies -207,Other humanities -217,Other Mathematics -188,Other Medical Sciences -172,Other Natural Sciences -232,Other Physical Sciences -202,Other social sciences -230,Particle and High-Energy Physics -210,Performance Evaluation and Benchmarking -233,Physical Chemistry -199,Political Science -194,Psychology -214,Pure Mathematics -212,Science and Engineering Education -200,Social and Economic Geography -197,Sociology -220,"Software Engineering, Systems, and Development" -209,Staff Activities (ACCESS or SP) -216,Statistics and Probability -251,Systematics and Population Biology -208,Training -191,Veterinary Science -222,Visualization and Human-Computer Systems diff --git a/tests/regression/data/jobs-metrics.csv b/tests/regression/data/jobs-metrics.csv deleted file mode 100644 index f5a9f14e..00000000 --- a/tests/regression/data/jobs-metrics.csv +++ /dev/null @@ -1,94 +0,0 @@ -id,label,description -avg_ace,ACCESS Credit Equivalents Charged: Per Job (SU),"The average amount of ACCESS Credit Equivalents charged per compute job.
- -The ACCESS Credit Equivalent is a measure of how much compute time was used on each resource. -One ACCESS Credit Equivalent is defined as one CPU Hour on SDSC Expanse (an AMD EPYC 7742 based compute resource). -The ACCESS Credit Equivalent allows comparison between usage of node-allocated, core-allocated and GPU-allocated -resources. It also allows a comparison between resources with different compute power per core. -The ACCESS allocations exchange calculator -lists conversion rates between an ACCESS Credit Equivalent and a service unit on a resource." -total_ace,ACCESS Credit Equivalents Charged: Total (SU),"The total amount of ACCESS Credit Equivalents charged.
- -The ACCESS Credit Equivalent is a measure of how much compute time was used on each resource. -One ACCESS Credit Equivalent is defined as one CPU Hour on SDSC Expanse (an AMD EPYC 7742 based compute resource). -The ACCESS Credit Equivalent allows comparison between usage of node-allocated, core-allocated and GPU-allocated -resources. It also allows a comparison between resources with different compute power per core. -The ACCESS allocations exchange calculator -lists conversion rates between an ACCESS Credit Equivalent and a service unit on a resource." -utilization,ACCESS Utilization (%),"The percentage of the ACCESS obligation of a resource that has been utilized by ACCESS jobs.
ACCESS Utilization: The ratio of the total CPU hours consumed by ACCESS jobs over a given time period divided by the total CPU hours that the system is contractually required to provide to ACCESS during that period. It does not include non-ACCESS jobs.
It is worth noting that this value is a rough estimate in certain cases where the resource providers don't provide accurate records of their system specifications, over time." -rate_of_usage,Allocation Usage Rate (XD SU/Hour),The rate of ACCESS allocation usage in XD SUs per hour. -rate_of_usage_ace,Allocation Usage Rate ACEs (SU/Hour),The rate of ACCESS allocation usage in ACCESS Credit Equivalents per hour. -avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per ACCESS job.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." -total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by ACCESS jobs.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." -max_processors,Job Size: Max (Core Count),The maximum size ACCESS job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. -min_processors,Job Size: Min (Core Count),The minimum size ACCESS job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. -normalized_avg_processors,Job Size: Normalized (% of Total Cores),The percentage average size ACCESS job over total machine cores.
Normalized Job Size: The percentage total number of processor cores used by a (parallel) job over the total number of cores on the machine. -avg_processors,Job Size: Per Job (Core Count),The average job size per ACCESS job.
Job Size: The number of processor cores used by a (parallel) job. -avg_job_size_weighted_by_ace,Job Size: Weighted By ACEs (Core Count),The average job size weighted by charge in ACCESS Credit Equivalents (ACEs). Defined as
Average Job Size Weighted By ACEs: sum(i = 0 to n){job i core count*job i charge in ACEs}/sum(i = 0 to n){job i charge in ACEs} -avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average ACCESS job size weighted by CPU Hours. Defined as
Average Job Size Weighted By CPU Hours: sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i = 0 to n){job i cpu hours} -avg_job_size_weighted_by_xd_su,Job Size: Weighted By XD SUs (Core Count),The average ACCESS job size weighted by charge in XD SUs. Defined as
Average Job Size Weighted By XD SUs: sum(i = 0 to n){job i core count*job i charge in xd sus}/sum(i = 0 to n){job i charge in xd sus} -avg_nu,NUs Charged: Per Job,"The average amount of NUs charged per ACCESS job.
-NU - Normalized Units: Roaming allocations are awarded in XSEDE Service Units (SUs). 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster. -For usage on a resource that is charged to a Roaming allocation, a normalization factor is applied. The normalization factor is based on -the method historically used to calculate 'Normalized Units' (Cray X-MP-equivalent SUs), which derives from a resource's performance on the HPL -benchmark.
- -Specifically, 1 Phase-1 DTF SU = 21.576 NUs, and the XD SU conversion factor for a resource is calculated by taking its NU conversion factor -and dividing it by 21.576. The standard formula for calculating a resource's NU conversion factor is: (Rmax * 1000 / 191) / P -where Rmax is the resource's Rmax result on the HPL benchmark in Gflops and P is the number of processors used in the benchmark. -In the absence of an HPL benchmark run, a conversion factor can be agreed upon, based on that of an architecturally similar platform -and scaled according to processor performance differences.
- -Conversion to Roaming SUs is handled by the XSEDE central accounting system, and RPs are only required to report usage in local SUs for all allocations.
- -Defining an SU charge for specialized compute resources (such as visualization hardware) or non-compute resources (such as storage) is possible, but there is no XSEDE-wide policy for doing so." -total_nu,NUs Charged: Total,"The total amount of NUs charged by ACCESS jobs.
-NU - Normalized Units: Roaming allocations are awarded in XSEDE Service Units (SUs). 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster. -For usage on a resource that is charged to a Roaming allocation, a normalization factor is applied. The normalization factor is based on -the method historically used to calculate 'Normalized Units' (Cray X-MP-equivalent SUs), which derives from a resource's performance on the HPL -benchmark.
- -Specifically, 1 Phase-1 DTF SU = 21.576 NUs, and the XD SU conversion factor for a resource is calculated by taking its NU conversion factor -and dividing it by 21.576. The standard formula for calculating a resource's NU conversion factor is: (Rmax * 1000 / 191) / P -where Rmax is the resource's Rmax result on the HPL benchmark in Gflops and P is the number of processors used in the benchmark. -In the absence of an HPL benchmark run, a conversion factor can be agreed upon, based on that of an architecturally similar platform -and scaled according to processor performance differences.
- -Conversion to Roaming SUs is handled by the XSEDE central accounting system, and RPs are only required to report usage in local SUs for all allocations.
- -Defining an SU charge for specialized compute resources (such as visualization hardware) or non-compute resources (such as storage) is possible, but there is no XSEDE-wide policy for doing so." -avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per ACCESS job. -total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by ACCESS jobs. -active_allocation_count,Number of Allocations: Active,The total number of funded projects that used ACCESS resources. -active_institution_count,Number of Institutions: Active,The total number of institutions that used ACCESS resources. -job_count,Number of Jobs Ended,The total number of ACCESS jobs that ended within the selected duration. -running_job_count,Number of Jobs Running,The total number of ACCESS jobs that are running. -started_job_count,Number of Jobs Started,The total number of ACCESS jobs that started executing within the selected duration. -submitted_job_count,Number of Jobs Submitted,The total number of ACCESS jobs that submitted/queued within the selected duration. -gateway_job_count,Number of Jobs via Gateway,"The total number of ACCESS jobs submitted through gateways (e.g., via a community user account) that ended within the selected duration.
Job: A scheduled process for a computer resource in a batch processing environment." -active_pi_count,Number of PIs: Active,The total number of PIs that used ACCESS resources. -active_resource_count,Number of Resources: Active,The total number of active ACCESS resources. -active_person_count,Number of Users: Active,The total number of users that used ACCESS resources. -expansion_factor,User Expansion Factor,"Gauging ACCESS job-turnaround time, it measures the ratio of wait time and the total time from submission to end of execution.
User Expansion Factor = ((wait duration + wall duration) / wall duration). " -avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a ACCESS job waits before execution on the designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." -total_waitduration_hours,Wait Hours: Total,"The total time, in hours, ACCESS jobs waited before execution on their designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." -avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.
In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." -total_wallduration_hours,Wall Hours: Total,"The total time, in hours, ACCESS jobs took to execute.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." -avg_su,XD SUs Charged: Per Job,"The average amount of XD SUs charged per ACCESS job.
-XD SU: 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster.
-SU - Service Units: Computational resources on the XSEDE are allocated and charged in service units (SUs). SUs are defined locally on each system, with conversion factors among systems based on HPL benchmark results.
- -Current TeraGrid supercomputers have complex multi-core and memory hierarchies. Each resource has a specific configuration that determines the number (N) of cores that can be dedicated to a job without slowing the code (and other user and system codes). Each resource defines for its system the minimum number of SUs charged for a job running in the default batch queue, calculated as wallclock runtime multiplied by N. Minimum charges may apply.
- -Note: The actual charge will depend on the specific requirements of the job (e.g., the mapping of the cores across the machine, or the priority you wish to obtain).
- -Note 2: The SUs show here have been normalized against the XSEDE Roaming service. Therefore they are comparable across resources." -total_su,XD SUs Charged: Total,"The total amount of XD SUs charged by ACCESS jobs.
-XD SU: 1 XSEDE SU is defined as one CPU-hour on a Phase-1 DTF cluster.
-SU - Service Units: Computational resources on the XSEDE are allocated and charged in service units (SUs). SUs are defined locally on each system, with conversion factors among systems based on HPL benchmark results.
- -Current TeraGrid supercomputers have complex multi-core and memory hierarchies. Each resource has a specific configuration that determines the number (N) of cores that can be dedicated to a job without slowing the code (and other user and system codes). Each resource defines for its system the minimum number of SUs charged for a job running in the default batch queue, calculated as wallclock runtime multiplied by N. Minimum charges may apply.
- -Note: The actual charge will depend on the specific requirements of the job (e.g., the mapping of the cores across the machine, or the priority you wish to obtain).
- -Note 2: The SUs show here have been normalized against the XSEDE Roaming service. Therefore they are comparable across resources." diff --git a/tests/regression/data/machine-learning-notebook-example-every-1000.csv b/tests/regression/data/machine-learning-notebook-example-every-1000.csv deleted file mode 100644 index 7f44b29e..00000000 --- a/tests/regression/data/machine-learning-notebook-example-every-1000.csv +++ /dev/null @@ -1,44 +0,0 @@ -,Nodes,Requested Wall Time,Wait Time,Wall Time,CPU User,"Mount point ""home"" data written","Mount point ""scratch"" data written",Total memory used -0,1,172800,11,506,,,, -1000,1,86400,1,66,,,, -2000,1,86400,18,752,,,, -3000,1,86400,8,5434,,,, -4000,1,86400,6,1572,,,, -5000,1,172800,7,2592,,,, -6000,1,14400,7,2800,,,, -7000,1,3600,2894,1357,,,, -8000,1,21600,116,7277,,,, -9000,1,21600,2173,6764,,,, -10000,1,21600,3574,7095,,,, -11000,1,9000,4,3564,88.01518903173182,992.4354304606816,267087841.6178405,811231171.4285715 -12000,1,21600,158,5565,,,, -13000,1,21600,59,6965,,,, -14000,1,21600,9,7760,,,, -15000,1,3600,22122,1335,,,, -16000,1,28800,130,9421,12.262731018331898,19749.432156075072,0,787292327.46875 -17000,1,28800,6,1990,,,, -18000,1,172800,13,73,,,, -19000,1,172800,7,129,,,, -20000,1,25200,4,25211,82.16279473845965,0,5113844.667942916,240912572.72941175 -21000,1,21600,18,6099,,,, -22000,1,21600,27,7131,,,, -23000,1,1800,61,1079,35.02319701051263,5818814744.200479,0,91742777.25 -24000,1,3600,5,2306,0.11814596015380158,,,33983854837.760006 -25000,1,960,1,59,2.025062333453586,0,0,118141168 -26000,1,172800,1,20494,87.54061396105656,548.1123048956289,0,224020798.15942028 -27000,4,7200,2,7214,99.2396948622311,441.0514348202534,34392345950.27519,1104895888.4 -28000,1,21600,13,55,1.2148444482641405,,, -29000,1,21600,171,40,,,, -30000,1,960,0,42,1.5504320217730077,0,0,112133180 -31000,1,1800,11,183,25.94758412119134,,,129784697856 -32000,1,21600,372,114,1.5571541609296509,,,92681043968 -33000,2,1800,134,139,55.875186246345784,533.6963000565588,754284385.8435647,137041136 -34000,1,7200,74,9,0.94096807333301,4681.666820975073,0,145688575 -35000,1,172800,22,83953,98.95217460976379,,,91787361316.73543 -36000,1,6000,8,152,0.4601673251104144,,,85277047466.66667 -37000,1,900,124,137,96.18834348033303,,,35703571797.333336 -38000,1,21600,12,56,24.892228849477622,,, -39000,1,21600,12,134,26.487756894710913,,,113801609216 -40000,1,21600,12,229,45.74138522053433,,,42761861802.66667 -41000,1,21600,20,307,0.9428384414161763,,,27184930360.88889 -42000,1,21600,130,386,1.68608777466353,,,49510031732.36363 diff --git a/tests/regression/data/realms.csv b/tests/regression/data/realms.csv deleted file mode 100644 index f0fb63e9..00000000 --- a/tests/regression/data/realms.csv +++ /dev/null @@ -1,8 +0,0 @@ -id,label -Accounts,Accounts -Allocations,Allocations -Cloud,Cloud -Gateways,Gateways -Jobs,Jobs -Requests,Requests -SUPREMM,SUPREMM diff --git a/tests/regression/data/xdmod-main-dev/jobs-2016-2017-month.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-month.csv new file mode 100644 index 00000000..ff9c7baa --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-month.csv @@ -0,0 +1,5 @@ +Time,CPU Hours: Total +2016-12-01,770775.9989 +2017-01-01,69779.8381 +2017-06-01,0.7911 +2017-07-01,43.6533 diff --git a/tests/regression/data/xdmod-main-dev/jobs-2016-2017-quarters.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-quarters.csv new file mode 100644 index 00000000..c3ae1193 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-quarters.csv @@ -0,0 +1,5 @@ +Time,CPU Hours: Total +2016-10-01,770775.9989 +2017-01-01,69779.8381 +2017-04-01,0.7911 +2017-07-01,43.6533 diff --git a/tests/regression/data/xdmod-main-dev/jobs-2016-2017-years.csv b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-years.csv new file mode 100644 index 00000000..d6faef19 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-2016-2017-years.csv @@ -0,0 +1,3 @@ +Time,CPU Hours: Total +2016-01-01,770775.9989 +2017-01-01,69824.2825 diff --git a/tests/regression/data/xdmod-main-dev/jobs-dimensions.csv b/tests/regression/data/xdmod-main-dev/jobs-dimensions.csv new file mode 100644 index 00000000..10887b04 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-dimensions.csv @@ -0,0 +1,17 @@ +id,label,description +none,None,Summarizes Jobs data reported to the Screwdriver database. +nsfdirectorate,Decanal Unit,Decanal Unit +parentscience,Department,Department +gpucount,GPU Count,A categorization of jobs into discrete groups based on the number of GPUs used by each job. +jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job. +jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited. +jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute. +nodecount,Node Count,A categorization of jobs into discrete groups based on node count. +pi,PI,The principal investigator of a project. +fieldofscience,PI Group,PI Group +qos,QOS,The quality of service of the job +queue,Queue,Queue pertains to the low level job queues on each resource. +resource,Resource,A resource is a remote computer that can run jobs. +resource_type,Resource Type,A categorization of resources into by their general capabilities. +username,System Username,The specific system username of the users who ran jobs. +person,User,"A person who is on a PIs allocation, hence able run jobs on resources." diff --git a/tests/regression/data/xdmod-main-dev/jobs-metrics.csv b/tests/regression/data/xdmod-main-dev/jobs-metrics.csv new file mode 100644 index 00000000..90c0bfe5 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-metrics.csv @@ -0,0 +1,27 @@ +id,label,description +avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." +total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." +avg_gpus,GPU Count: Per Job,The average job size per Screwdriver job.
Job Size: The number of GPUs used by a (parallel) job. +avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours." +total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours." +max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. +min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. +normalized_avg_processors,Job Size: Normalized (% of Total Cores),"The percentage average size Screwdriver job divided by the total number of cores in the resource where the job ran. The job normalization calculation assumes that the resource size is constant. This statistic should not be used with a time range where the resource size changes, because the statistic will be incorrect.
Normalized Job Size: The ratio of the total number of processor cores used by a (parallel) job over the total number of cores on the resource." +avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job. +avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as
Average Job Size Weighted By CPU Hours: sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i = 0 to n){job i cpu hours} +avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as
Average Job Size Weighted By GPU Hours: sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i = 0 to n){job i gpu hours} +avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per Screwdriver job. +total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by Screwdriver jobs. +job_count,Number of Jobs Ended,The total number of Screwdriver jobs that ended within the selected duration. +running_job_count,Number of Jobs Running,The total number of Screwdriver jobs that are running. +started_job_count,Number of Jobs Started,The total number of Screwdriver jobs that started executing within the selected duration. +submitted_job_count,Number of Jobs Submitted,The total number of Screwdriver jobs that submitted/queued within the selected duration. +active_pi_count,Number of PIs: Active,The total number of PIs that used Screwdriver resources. +active_resource_count,Number of Resources: Active,The total number of active Screwdriver resources. +active_person_count,Number of Users: Active,The total number of users that used Screwdriver resources. +utilization,Screwdriver Utilization (%),The percentage of core time that a resource has been running jobs.
Screwdriver Utilization: The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resource and the percent of the resource allocated through Screwdriver). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications. +expansion_factor,User Expansion Factor,The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job. +avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a Screwdriver job waits before execution on the designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." +total_waitduration_hours,Wait Hours: Total,"The total time, in hours, Screwdriver jobs waited before execution on their designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." +avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.
In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." +total_wallduration_hours,Wall Hours: Total,"The total time, in hours, Screwdriver jobs took to execute.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." diff --git a/tests/regression/data/xdmod-main-dev/jobs-pi-group-filter-values.csv b/tests/regression/data/xdmod-main-dev/jobs-pi-group-filter-values.csv new file mode 100644 index 00000000..a31287cc --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/jobs-pi-group-filter-values.csv @@ -0,0 +1,41 @@ +id,label +53,Algebra and Number Theory +10,Arts +44,Biochemistry and Molecular Structure and Function +45,Biophysics +47,Cell Biology +127,Computer and Computation Theory +61,"Decision, Risk, and Management Science" +99,Design and Computer-Integrated Engineering +139,"Design, Tools, and Test" +55,Economics +97,Emerging Technologies Initiation +137,Experimental Systems +112,Extragalactic Astronomy and Cosmology +88,"Fluid, Particulate, and Hydraulic Systems" +115,Galactic Astronomy +68,Geology and Paleontology +72,Geophysics +65,Global Atmospheric Research +60,Law and Social Sciences +91,Mechanics and Materials +126,"Metals, Ceramics, and Electronic Materials" +98,Operations Research and Production Systems +122,Organic and Macromolecular Chemistry +123,Physical Chemistry +81,Polar Aeronomy and Astrophysics +84,Polar Meteorology +59,Polar Ocean and Climate Systems +93,"Quantum Electronics, Waves, and Beams" +71,Seismology +57,Sociology +124,Solid State Chemistry and Polymers +94,Solid-State and Microstructures +69,Statistics and Probability +114,Stellar Astronomy and Astrophysics +90,Structures and Building Systems +49,Systematic and Population Biology +138,Systems Prototyping and Fabrication +70,Tectonics +119,Theoretical Physics +74,Volcanology and Mantle Geochemistry diff --git a/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv new file mode 100644 index 00000000..758d84d3 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-no-fields-no-filters.csv @@ -0,0 +1,56 @@ +,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group +0,6110386,Robertson,UTC,a19290db301c97aa78a57524bc42c8c78bf455de,"Harrier, Hen",Screwdriver,curry,1483118438,1483118438,1483118443,1483118438,1,1,0,-1,5,0,5,0,0:0,COMPLETED,1,48000Mn,172800,white,Computer and Information Science and Engineering,Computer and Computation Research,Computer and Computation Theory +1000,970348,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483064151,1483064152,1483064172,1483064151,1,8,0,-1,20,1,160,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +2000,981745,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483102502,1483102504,1483102531,1483102502,1,8,0,-1,27,2,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +3000,6112283[436],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483135648,1483139913,1483139937,1483135648,1,1,0,-1,24,4265,24,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +4000,178502,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116701,1483130204,1483130230,1483116701,1,8,0,-1,26,13503,208,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +5000,974487,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483077974,1483077974,1483078015,1483077974,1,8,0,-1,41,0,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +6000,982727,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483106505,1483106505,1483106556,1483106505,1,8,0,-1,51,0,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +7000,2281895,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483115132,1483115132,1483116616,1483115132,1,12,0,-1,1484,0,17808,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +8000,972196,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483070299,1483070301,1483070337,1483070299,1,8,0,-1,36,2,288,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +9000,978066,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483088977,1483088978,1483089027,1483088977,1,8,0,-1,49,1,392,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +10000,983035,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483107862,1483107863,1483108007,1483107862,1,8,0,-1,144,1,1152,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +11000,1009935,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483130897,1483130908,1483131130,1483130897,1,8,0,-1,222,11,1776,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +12000,2284391,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483137557,1483137558,1483139202,1483137557,1,12,0,-1,1644,1,19728,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +13000,1005349,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483071422,1483071534,1483071778,1483071422,1,8,0,-1,244,112,1952,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +14000,1007824,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483105333,1483105429,1483105948,1483105333,1,8,0,-1,519,96,4152,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +15000,1010629,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483140307,1483140400,1483140550,1483140307,1,8,0,-1,150,93,1200,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +16000,6111037[473],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483125912,1483129603,1483129747,1483125913,1,1,0,-1,144,3691,144,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +17000,988528,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483119249,1483125280,1483125321,1483119249,1,8,0,-1,41,6031,328,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +18000,991993,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483131196,1483136908,1483136994,1483131196,1,8,0,-1,86,5712,688,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +19000,1008740,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483117402,1483124540,1483126240,1483117402,1,12,0,-1,1700,7138,20400,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +20000,2276090,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483060299,1483060299,1483062513,1483060299,1,12,0,-1,2214,0,26568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +21000,967122,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483053110,1483053111,1483056506,1483053110,1,8,0,-1,3395,1,27160,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +22000,2283293,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483126817,1483126818,1483128722,1483126817,1,12,0,-1,1904,1,22848,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +23000,1008638,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116706,1483122377,1483124429,1483116706,1,12,0,-1,2052,5671,24624,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +24000,972040,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483069980,1483069981,1483074019,1483069980,1,8,0,-1,4038,1,32304,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +25000,991884,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483130901,1483136404,1483141877,1483130901,1,8,0,-1,5473,5503,43784,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +26000,6105349,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1482948669,1482948735,1483066377,1482948669,1,1,0,-1,117642,66,117642,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication +27000,1005309,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483208573,1483209389,1483209414,1483208573,1,8,0,-1,25,816,200,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +28000,6114243[425],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483202456,1483205646,1483205659,1483202456,1,1,0,-1,13,3190,13,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +29000,998101,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483159238,1483159264,1483154003,1,8,0,-1,26,5235,208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +30000,184276,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158935,1483191164,1483191182,1483158935,1,8,0,-1,18,32229,144,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +31000,2292895,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483183969,1483183969,1483185436,1483183969,1,12,0,-1,1467,0,17604,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +32000,2302043,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483211208,1483211208,1483212911,1483211208,1,12,0,-1,1703,0,20436,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +33000,2286102,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154003,1483154004,1483155247,1483154003,1,12,0,-1,1243,1,14916,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +34000,2295330,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483191135,1483191136,1483192490,1483191135,1,12,0,-1,1354,1,16248,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +35000,2304491,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483218813,1483218814,1483219999,1483218813,1,12,0,-1,1185,1,14220,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +36000,1004397,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483206039,1483207040,1483207079,1483206039,1,8,0,-1,39,1001,312,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +37000,1007907,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483216840,1483217894,1483217930,1483216840,1,8,0,-1,36,1054,288,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +38000,189492,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483219052,1483220591,1483220646,1483219052,1,8,0,-1,55,1539,440,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +39000,1014185,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483177983,1483178402,1483178494,1483177983,1,8,0,-1,92,419,736,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +40000,6112860,Robertson,UTC,7cdc338fbe4024d633ba19deca41e22c78704105,Dunlin,Screwdriver,curry,1483142479,1483143257,1483143695,1483142479,1,12,0,-1,438,778,5256,0,0:0,COMPLETED,12,2147486448Mn,540,white,Biological Sciences,Environmental Biology,Systematic and Population Biology +41000,993914,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483137599,1483144206,1483144241,1483137599,1,8,0,-1,35,6607,280,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +42000,997151,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483150168,1483156370,1483156401,1483150168,1,8,0,-1,31,6202,248,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +43000,1000391,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483160667,1483167946,1483167986,1483160667,1,8,0,-1,40,7279,320,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +44000,1003122,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483172712,1483182324,1483182384,1483172712,1,8,0,-1,60,9612,480,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +45000,183442,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483151833,1483184199,1483184231,1483151833,1,8,0,-1,32,32366,256,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +46000,2288079,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483166466,1483166466,1483168335,1483166466,1,12,0,-1,1869,0,22428,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +47000,2300772,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207320,1483207320,1483209262,1483207320,1,12,0,-1,1942,0,23304,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +48000,2290078,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483174731,1483174734,1483177171,1483174731,1,12,0,-1,2437,3,29244,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +49000,2304088,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483217556,1483217557,1483219647,1483217556,1,12,0,-1,2090,1,25080,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +50000,1011153,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483146809,1483157125,1483159016,1483146809,1,12,0,-1,1891,10316,22692,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +51000,1006854,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483213460,1483214511,1483219230,1483213460,1,8,0,-1,4719,1051,37752,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +52000,6106621,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1483007306,1483150677,1483157807,1483007306,1,8,0,-1,7130,143371,57040,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +53000,6110124,Robertson,UTC,a3a4db35c34bc1ae71d8ba76352b66796cddb74e,"Grey, Lesser",Screwdriver,curry,1483117470,1483124196,1483154281,1483117470,1,1,0,-1,30085,6726,30085,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics +54000,6107522,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1483033412,1483033544,1483158212,1483033412,1,1,0,-1,124668,132,124668,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication diff --git a/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv new file mode 100644 index 00000000..0685e971 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/raw-data-every-1000-with-fields-and-filters.csv @@ -0,0 +1,35 @@ +,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department +0,968193,banana-cream,1483056132,0,Social and Economic Science +1000,971442,banana-cream,1483068165,0,Social and Economic Science +2000,984175,banana-cream,1483112456,0,Social and Economic Science +3000,993014,banana-cream,1483140821,0,Social and Economic Science +4000,976375,banana-cream,1483084157,0,Social and Economic Science +5000,983957,banana-cream,1483111812,0,Social and Economic Science +6000,2284603,banana-cream,1483139473,0,Social and Economic Science +7000,973762,banana-cream,1483075149,0,Social and Economic Science +8000,979555,banana-cream,1483093851,0,Social and Economic Science +9000,984311,banana-cream,1483112900,0,Social and Economic Science +10000,2283845,banana-cream,1483132302,0,Social and Economic Science +11000,988946,banana-cream,1483126657,0,Social and Economic Science +12000,992368,banana-cream,1483138292,0,Social and Economic Science +13000,2279980,banana-cream,1483097026,0,Social and Economic Science +14000,2278883,banana-cream,1483086983,0,Social and Economic Science +15000,979689,banana-cream,1483094414,0,Social and Economic Science +16000,988604,banana-cream,1483125536,0,Social and Economic Science +17000,1008884,banana-cream,1483220332,0,Social and Economic Science +18000,999879,banana-cream,1483165819,0,Social and Economic Science +19000,2292962,banana-cream,1483184182,0,Social and Economic Science +20000,2302093,banana-cream,1483211322,0,Social and Economic Science +21000,2288974,banana-cream,1483170354,0,Social and Economic Science +22000,2298407,banana-cream,1483200279,0,Social and Economic Science +23000,2306930,banana-cream,1483226400,0,Social and Economic Science +24000,1006973,banana-cream,1483214989,0,Social and Economic Science +25000,1010648,banana-cream,1483226156,0,Social and Economic Science +26000,995617,banana-cream,1483151188,0,Social and Economic Science +27000,999309,banana-cream,1483163285,0,Social and Economic Science +28000,1002056,banana-cream,1483177400,0,Social and Economic Science +29000,2289840,banana-cream,1483173994,0,Social and Economic Science +30000,2303857,banana-cream,1483216696,0,Social and Economic Science +31000,2293260,banana-cream,1483185052,0,Social and Economic Science +32000,1005259,banana-cream,1483209237,0,Social and Economic Science +33000,998935,banana-cream,1483161891,0,Social and Economic Science diff --git a/tests/regression/data/xdmod-main-dev/realms.csv b/tests/regression/data/xdmod-main-dev/realms.csv new file mode 100644 index 00000000..e6cc8eb1 --- /dev/null +++ b/tests/regression/data/xdmod-main-dev/realms.csv @@ -0,0 +1,5 @@ +id,label +Cloud,Cloud +Jobs,Jobs +ResourceSpecifications,Resource Specifications +Storage,Storage diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv new file mode 100644 index 00000000..ff9c7baa --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-month.csv @@ -0,0 +1,5 @@ +Time,CPU Hours: Total +2016-12-01,770775.9989 +2017-01-01,69779.8381 +2017-06-01,0.7911 +2017-07-01,43.6533 diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv new file mode 100644 index 00000000..c3ae1193 --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-quarters.csv @@ -0,0 +1,5 @@ +Time,CPU Hours: Total +2016-10-01,770775.9989 +2017-01-01,69779.8381 +2017-04-01,0.7911 +2017-07-01,43.6533 diff --git a/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv b/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv new file mode 100644 index 00000000..d6faef19 --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-2016-2017-years.csv @@ -0,0 +1,3 @@ +Time,CPU Hours: Total +2016-01-01,770775.9989 +2017-01-01,69824.2825 diff --git a/tests/regression/data/xdmod10.5/jobs-dimensions.csv b/tests/regression/data/xdmod10.5/jobs-dimensions.csv new file mode 100644 index 00000000..10887b04 --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-dimensions.csv @@ -0,0 +1,17 @@ +id,label,description +none,None,Summarizes Jobs data reported to the Screwdriver database. +nsfdirectorate,Decanal Unit,Decanal Unit +parentscience,Department,Department +gpucount,GPU Count,A categorization of jobs into discrete groups based on the number of GPUs used by each job. +jobsize,Job Size,A categorization of jobs into discrete groups based on the number of cores used by each job. +jobwaittime,Job Wait Time,A categorization of jobs into discrete groups based on the total linear time each job waited. +jobwalltime,Job Wall Time,A categorization of jobs into discrete groups based on the total linear time each job took to execute. +nodecount,Node Count,A categorization of jobs into discrete groups based on node count. +pi,PI,The principal investigator of a project. +fieldofscience,PI Group,PI Group +qos,QOS,The quality of service of the job +queue,Queue,Queue pertains to the low level job queues on each resource. +resource,Resource,A resource is a remote computer that can run jobs. +resource_type,Resource Type,A categorization of resources into by their general capabilities. +username,System Username,The specific system username of the users who ran jobs. +person,User,"A person who is on a PIs allocation, hence able run jobs on resources." diff --git a/tests/regression/data/xdmod10.5/jobs-metrics.csv b/tests/regression/data/xdmod10.5/jobs-metrics.csv new file mode 100644 index 00000000..2f4c6d7e --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-metrics.csv @@ -0,0 +1,27 @@ +id,label,description +avg_cpu_hours,CPU Hours: Per Job,"The average CPU hours (number of CPU cores x wall time hours) per Screwdriver job.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." +total_cpu_hours,CPU Hours: Total,"The total CPU hours (number of CPU cores x wall time hours) used by Screwdriver jobs.
For each job, the CPU usage is aggregated. For example, if a job used 1000 CPUs for one minute, it would be aggregated as 1000 CPU minutes or 16.67 CPU hours." +avg_gpus,GPU Count: Per Job,The average job size per Screwdriver job.
Job Size: The number of GPUs used by a (parallel) job. +avg_gpu_hours,GPU Hours: Per Job,"The average GPU hours (number of GPU cores x wall time hours) per Screwdriver job.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours." +total_gpu_hours,GPU Hours: Total,"The total GPU hours (number of GPUs x wall time hours) used by Screwdriver jobs.
For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours." +max_processors,Job Size: Max (Core Count),The maximum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. +min_processors,Job Size: Min (Core Count),The minimum size Screwdriver job in number of cores.
Job Size: The total number of processor cores used by a (parallel) job. +normalized_avg_processors,Job Size: Normalized (% of Total Cores),The percentage average size Screwdriver job over total machine cores.
Normalized Job Size: The percentage total number of processor cores used by a (parallel) job over the total number of cores on the machine. +avg_processors,Job Size: Per Job (Core Count),The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job. +avg_job_size_weighted_by_cpu_hours,Job Size: Weighted By CPU Hours (Core Count),The average Screwdriver job size weighted by CPU Hours. Defined as
Average Job Size Weighted By CPU Hours: sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i = 0 to n){job i cpu hours} +avg_job_size_weighted_by_gpu_hours,Job Size: Weighted By GPU Hours (GPU Count),The average Screwdriver job size weighted by GPU Hours. Defined as
Average Job Size Weighted By GPU Hours: sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i = 0 to n){job i gpu hours} +avg_node_hours,Node Hours: Per Job,The average node hours (number of nodes x wall time hours) per Screwdriver job. +total_node_hours,Node Hours: Total,The total node hours (number of nodes x wall time hours) used by Screwdriver jobs. +job_count,Number of Jobs Ended,The total number of Screwdriver jobs that ended within the selected duration. +running_job_count,Number of Jobs Running,The total number of Screwdriver jobs that are running. +started_job_count,Number of Jobs Started,The total number of Screwdriver jobs that started executing within the selected duration. +submitted_job_count,Number of Jobs Submitted,The total number of Screwdriver jobs that submitted/queued within the selected duration. +active_pi_count,Number of PIs: Active,The total number of PIs that used Screwdriver resources. +active_resource_count,Number of Resources: Active,The total number of active Screwdriver resources. +active_person_count,Number of Users: Active,The total number of users that used Screwdriver resources. +utilization,Screwdriver Utilization (%),The percentage of core time that a resource has been running jobs.
Screwdriver Utilization: The ratio of the total CPU hours consumed by jobs over a given time period divided by the maximum CPU hours that the system could deliver (based on the number of cores present on the resources). This value does not take into account downtimes or outages. It is just calculated based on the number of cores in the resource specifications. +expansion_factor,User Expansion Factor,The average job size per Screwdriver job.
Job Size: The number of processor cores used by a (parallel) job. +avg_waitduration_hours,Wait Hours: Per Job,"The average time, in hours, a Screwdriver job waits before execution on the designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." +total_waitduration_hours,Wait Hours: Total,"The total time, in hours, Screwdriver jobs waited before execution on their designated resource.
Wait Time: Wait time is defined as the linear time between submission of a job by a user until it begins to execute." +avg_wallduration_hours,Wall Hours: Per Job,"The average time, in hours, a job takes to execute.
In timeseries view mode, the statistic shows the average wall time per job per time period. In aggregate view mode the statistic only includes the job wall hours between the defined time range. The wall hours outside the time range are not included in the calculation.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." +total_wallduration_hours,Wall Hours: Total,"The total time, in hours, Screwdriver jobs took to execute.
Wall Time: Wall time is defined as the linear time between start and end time of execution for a particular job." diff --git a/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv b/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv new file mode 100644 index 00000000..a31287cc --- /dev/null +++ b/tests/regression/data/xdmod10.5/jobs-pi-group-filter-values.csv @@ -0,0 +1,41 @@ +id,label +53,Algebra and Number Theory +10,Arts +44,Biochemistry and Molecular Structure and Function +45,Biophysics +47,Cell Biology +127,Computer and Computation Theory +61,"Decision, Risk, and Management Science" +99,Design and Computer-Integrated Engineering +139,"Design, Tools, and Test" +55,Economics +97,Emerging Technologies Initiation +137,Experimental Systems +112,Extragalactic Astronomy and Cosmology +88,"Fluid, Particulate, and Hydraulic Systems" +115,Galactic Astronomy +68,Geology and Paleontology +72,Geophysics +65,Global Atmospheric Research +60,Law and Social Sciences +91,Mechanics and Materials +126,"Metals, Ceramics, and Electronic Materials" +98,Operations Research and Production Systems +122,Organic and Macromolecular Chemistry +123,Physical Chemistry +81,Polar Aeronomy and Astrophysics +84,Polar Meteorology +59,Polar Ocean and Climate Systems +93,"Quantum Electronics, Waves, and Beams" +71,Seismology +57,Sociology +124,Solid State Chemistry and Polymers +94,Solid-State and Microstructures +69,Statistics and Probability +114,Stellar Astronomy and Astrophysics +90,Structures and Building Systems +49,Systematic and Population Biology +138,Systems Prototyping and Fabrication +70,Tectonics +119,Theoretical Physics +74,Volcanology and Mantle Geochemistry diff --git a/tests/regression/data/xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv b/tests/regression/data/xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv new file mode 100644 index 00000000..f830643e --- /dev/null +++ b/tests/regression/data/xdmod10.5/raw-data-every-1000-no-fields-no-filters.csv @@ -0,0 +1,56 @@ +,Local Job Id,Resource,Timezone,System Username (Deidentified),User,Organization,Quality of Service,Submit Time (Timestamp),Start Time (Timestamp),End Time (Timestamp),Eligible Time (Timestamp),Nodes,Cores,GPUs,Memory Used,Wall Time,Wait Time,Core Time,GPU Time,Exit Code,Exit State,Requested Cores,Requested memory,Requested Wall Time,Queue,Decanal Unit,Department,PI Group +0,977669,Posidriv,UTC,2e73831287e6f71196828b70000ec53828d6148b,"Dove, Turtle",Screwdriver,apple,1482356614,1482390210,1483069389,1482356614,1,12,0,-1,679179,33596,8150148,0,0:0,COMPLETED,12,48000Mn,10800000,croutons,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +1000,6106087,Robertson,UTC,ea56fb19a839cb996b0e014f70d35a03606aa19f,Moorhen,Screwdriver,curry,1482980998,1482981066,1483080831,1482980998,1,1,0,-1,99765,68,99765,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Computer and Information Science and Engineering,Microelectronic Information Processing Systems,Systems Prototyping and Fabrication +2000,975068,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483080229,1483080229,1483080249,1483080229,1,8,0,-1,20,0,160,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +3000,974176,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076858,1483076859,1483076886,1483076858,1,8,0,-1,27,1,216,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +4000,985572,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483116354,1483116924,1483116945,1483116354,1,8,0,-1,21,570,168,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +5000,990168,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483124951,1483130983,1483131006,1483124951,1,8,0,-1,23,6032,184,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +6000,969126,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483059498,1483059498,1483059563,1483059498,1,8,0,-1,65,0,520,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +7000,978262,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483089659,1483089659,1483089703,1483089659,1,8,0,-1,44,0,352,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +8000,171798,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483083351,1483083351,1483083386,1483083351,1,8,0,-1,35,0,280,0,1:0,FAILED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +9000,968562,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483057298,1483057299,1483057345,1483057298,1,8,0,-1,46,1,368,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +10000,974174,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076846,1483076847,1483076913,1483076846,1,8,0,-1,66,1,528,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +11000,979960,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483095351,1483095353,1483095404,1483095351,1,8,0,-1,51,2,408,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +12000,984736,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483114441,1483114442,1483114544,1483114441,1,8,0,-1,102,1,816,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +13000,2277762,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483076479,1483076480,1483078267,1483076479,1,12,0,-1,1787,1,21444,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +14000,6112330,Robertson,UTC,c73e24b5c1a6de69db0bd9bf67fb461cf4da104f,Lapwing,Screwdriver,curry,1483136546,1483136628,1483138226,1483136546,1,1,0,-1,1598,82,1598,0,0:0,CANCELLED by 0,1,8Gn,36000,white,Engineering,Design and Manufacturing Systems,Operations Research and Production Systems +15000,1006230,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483083136,1483083193,1483083332,1483083136,1,8,0,-1,139,57,1112,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +16000,1008815,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483118123,1483118171,1483118309,1483118123,1,8,0,-1,138,48,1104,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +17000,6111037[445],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483125912,1483129321,1483129510,1483125913,1,1,0,-1,189,3409,189,0,0:0,COMPLETED,1,2147487648Mn,900,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +18000,6112283[353],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483135648,1483139250,1483139291,1483135648,1,1,0,-1,41,3602,41,0,0:0,COMPLETED,1,48000Mn,10800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +19000,989648,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483123196,1483129202,1483129233,1483123196,1,8,0,-1,31,6006,248,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +20000,993076,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483134403,1483140976,1483141013,1483134403,1,8,0,-1,37,6573,296,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +21000,6082994,Robertson,UTC,d2e5a44089d3217a3c4538f936d800471b145a4c,"Thrush, Swainson's",Screwdriver,curry,1482264225,1483074239,1483076033,1482264225,1,4,0,-1,1794,810014,7176,0,0:0,COMPLETED,4,125Gn,7200,white,Mathematical and Physical Sciences,Astronomical Sciences,Galactic Astronomy +22000,2279431,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483092109,1483092109,1483094048,1483092109,1,12,0,-1,1939,0,23268,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +23000,2277824,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483077039,1483077040,1483079075,1483077039,1,12,0,-1,2035,1,24420,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +24000,1003828,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483050771,1483057342,1483059250,1483050771,1,12,0,-1,1908,6571,22896,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +25000,6083488,Robertson,UTC,d2e5a44089d3217a3c4538f936d800471b145a4c,"Thrush, Swainson's",Screwdriver,curry,1482264356,1483114035,1483115957,1482264356,1,4,0,-1,1922,849679,7688,0,0:0,COMPLETED,4,125Gn,7200,white,Mathematical and Physical Sciences,Astronomical Sciences,Galactic Astronomy +26000,984588,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483114013,1483114015,1483118919,1483114013,1,8,0,-1,4904,2,39232,0,1:0,FAILED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +27000,6106282,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1482988989,1483091687,1483096295,1482988989,1,8,0,-1,4608,102698,36864,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +28000,6110529,Robertson,UTC,a3a4db35c34bc1ae71d8ba76352b66796cddb74e,"Grey, Lesser",Screwdriver,curry,1483118933,1483140830,1483172555,1483118933,1,1,0,-1,31725,21897,31725,0,0:0,COMPLETED,1,2147486448Mn,259200,white,Geosciences,Earth Sciences,Tectonics +29000,1005802,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483210021,1483210931,1483210958,1483210021,1,8,0,-1,27,910,216,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +30000,186979,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207102,1483209435,1483209451,1483207102,1,8,0,-1,16,2333,128,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +31000,998331,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483154514,1483159821,1483159839,1483154514,1,8,0,-1,18,5307,144,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +32000,184674,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483161882,1483195550,1483195576,1483161882,1,8,0,-1,26,33668,208,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +33000,2293381,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483185388,1483185388,1483187114,1483185388,1,12,0,-1,1726,0,20712,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +34000,2302343,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483212133,1483212133,1483213855,1483212133,1,12,0,-1,1722,0,20664,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +35000,2286627,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158678,1483158679,1483159873,1483158678,1,12,0,-1,1194,1,14328,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +36000,2296175,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483193616,1483193617,1483195331,1483193616,1,12,0,-1,1714,1,20568,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +37000,2304947,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483220143,1483220144,1483221600,1483220143,1,12,0,-1,1456,1,17472,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +38000,1004771,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483207312,1483207904,1483207951,1483207312,1,8,0,-1,47,592,376,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +39000,1008199,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483217878,1483218733,1483218884,1483217878,1,8,0,-1,151,855,1208,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +40000,1010919,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483143792,1483143885,1483144233,1483143792,1,8,0,-1,348,93,2784,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +41000,1014861,Posidriv,UTC,cdd144ba0c808c63ec0bbdeb890cd90b74587c9b,"Bunting, Reed",Screwdriver,banana-cream,1483180668,1483180806,1483180917,1483180668,1,8,0,-1,111,138,888,0,0:0,COMPLETED,8,2147486448Mn,259200,black,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +42000,6113756[245],Robertson,UTC,420f3254a8b357efce12369d50ff33dc437f4f0b,"Grey, Great",Screwdriver,curry,1483199295,1483201396,1483201474,1483199297,1,1,0,-1,78,2101,78,0,0:0,COMPLETED,1,2147531648Mn,1800,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,"Decision, Risk, and Management Science" +43000,994128,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483138376,1483145047,1483145213,1483138376,1,8,0,-1,166,6671,1328,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +44000,997525,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483151518,1483157350,1483157384,1483151518,1,8,0,-1,34,5832,272,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +45000,1000594,Mortorq,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483161348,1483168897,1483168986,1483161348,1,8,0,-1,89,7549,712,0,0:0,COMPLETED,8,2147486848Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +46000,1010026,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483132282,1483143164,1483144904,1483132282,1,12,0,-1,1740,10882,20880,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +47000,184170,Phillips,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483158181,1483190013,1483190125,1483158181,1,8,0,-1,112,31832,896,0,0:0,COMPLETED,8,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +48000,2288942,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483170250,1483170250,1483172607,1483170250,1,12,0,-1,2357,0,28284,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +49000,2302966,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483214056,1483214056,1483216359,1483214056,1,12,0,-1,2303,0,27636,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +50000,2291733,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483180588,1483180589,1483182543,1483180588,1,12,0,-1,1954,1,23448,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +51000,2306026,Frearson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483223552,1483223553,1483225424,1483223552,1,12,0,-1,1871,1,22452,0,0:0,COMPLETED,12,2147486648Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +52000,1014640,Posidriv,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,banana-cream,1483179697,1483191989,1483193818,1483179697,1,12,0,-1,1829,12292,21948,0,0:0,COMPLETED,12,2147486448Mn,108000,black,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology +53000,1012058,Posidriv,UTC,2e73831287e6f71196828b70000ec53828d6148b,"Dove, Turtle",Screwdriver,apple,1483157875,1483157966,1483167364,1483157875,1,8,0,-1,9398,91,75184,0,0:0,COMPLETED,8,23000Mn,10800000,croutons,Mathematical and Physical Sciences,Chemistry,Physical Chemistry +54000,6108290,Robertson,UTC,0d3a3148ee4a6b664af068bc84bf862f01d8732d,Honey-buzzard,Screwdriver,curry,1483054131,1483207468,1483213641,1483054131,1,8,0,-1,6173,153337,49384,0,0:0,COMPLETED,8,2147486448Mn,108000,white,"Social, Behavioral, and Economic Sciences",Social and Economic Science,Sociology diff --git a/tests/regression/data/xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv b/tests/regression/data/xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv new file mode 100644 index 00000000..5c9e0675 --- /dev/null +++ b/tests/regression/data/xdmod10.5/raw-data-every-1000-with-fields-and-filters.csv @@ -0,0 +1,35 @@ +,Local Job Id,Quality of Service,Start Time (Timestamp),GPUs,Department +0,2286873,banana-cream,1483160526,0,Social and Economic Science +1000,2293183,banana-cream,1483184848,0,Social and Economic Science +2000,2302227,banana-cream,1483211690,0,Social and Economic Science +3000,2289494,banana-cream,1483172442,0,Social and Economic Science +4000,2299179,banana-cream,1483202557,0,Social and Economic Science +5000,2284778,banana-cream,1483141002,0,Social and Economic Science +6000,2296200,banana-cream,1483193678,0,Social and Economic Science +7000,2287312,banana-cream,1483163089,0,Social and Economic Science +8000,2300378,banana-cream,1483206150,0,Social and Economic Science +9000,2280848,banana-cream,1483105401,0,Social and Economic Science +10000,2281978,banana-cream,1483115753,0,Social and Economic Science +11000,2281388,banana-cream,1483110374,0,Social and Economic Science +12000,2280890,banana-cream,1483105946,0,Social and Economic Science +13000,1008307,banana-cream,1483219024,0,Social and Economic Science +14000,998351,banana-cream,1483159887,0,Social and Economic Science +15000,1006328,banana-cream,1483212749,0,Social and Economic Science +16000,1009986,banana-cream,1483223948,0,Social and Economic Science +17000,994932,banana-cream,1483148678,0,Social and Economic Science +18000,998665,banana-cream,1483160915,0,Social and Economic Science +19000,1001452,banana-cream,1483173813,0,Social and Economic Science +20000,1003324,boysenberry,1483204008,0,Social and Economic Science +21000,1002994,banana-cream,1483181740,0,Social and Economic Science +22000,971318,banana-cream,1483067878,0,Social and Economic Science +23000,983519,banana-cream,1483109661,0,Social and Economic Science +24000,992780,banana-cream,1483139789,0,Social and Economic Science +25000,976041,banana-cream,1483083428,0,Social and Economic Science +26000,983897,banana-cream,1483111546,0,Social and Economic Science +27000,973004,banana-cream,1483072678,0,Social and Economic Science +28000,978764,banana-cream,1483091343,0,Social and Economic Science +29000,983635,banana-cream,1483110039,0,Social and Economic Science +30000,988096,banana-cream,1483123810,0,Social and Economic Science +31000,991564,banana-cream,1483135262,0,Social and Economic Science +32000,973969,banana-cream,1483076149,0,Social and Economic Science +33000,987124,banana-cream,1483120872,0,Social and Economic Science diff --git a/tests/regression/data/xdmod10.5/realms.csv b/tests/regression/data/xdmod10.5/realms.csv new file mode 100644 index 00000000..5e555121 --- /dev/null +++ b/tests/regression/data/xdmod10.5/realms.csv @@ -0,0 +1,4 @@ +id,label +Cloud,Cloud +Jobs,Jobs +Storage,Storage diff --git a/tests/regression/test_datawarehouse_regression.py b/tests/regression/test_datawarehouse_regression.py index 437f5821..dde44fc3 100644 --- a/tests/regression/test_datawarehouse_regression.py +++ b/tests/regression/test_datawarehouse_regression.py @@ -1,14 +1,17 @@ +from datetime import date from dotenv import load_dotenv import numpy +import os from os.path import dirname, expanduser import pandas from pathlib import Path import pytest from xdmod_data.warehouse import DataWarehouse -XDMOD_URL = 'https://xdmod.access-ci.org' +XDMOD_HOST = os.environ['XDMOD_HOST'] +XDMOD_VERSION = os.environ['XDMOD_VERSION'] TOKEN_PATH = '~/.xdmod-data-token' -DATA_DIR = dirname(__file__) + '/data' +DATA_DIR = dirname(__file__) + '/data/' + XDMOD_VERSION load_dotenv(Path(expanduser(TOKEN_PATH)), override=True) @@ -16,7 +19,7 @@ @pytest.fixture(scope='module') def valid_dw(): - with DataWarehouse(XDMOD_URL) as dw: + with DataWarehouse(XDMOD_HOST) as dw: yield dw @@ -25,46 +28,77 @@ def __assert_dfs_equal( actual, dtype='object', index_col='id', + columns_name=None, ): - expected = pandas.read_csv( - DATA_DIR + '/' + data_file, - dtype=dtype, - index_col=index_col, - keep_default_na=False, - na_values=[''], - ).fillna(numpy.nan) - expected.columns = expected.columns.astype('string') - assert expected.equals(actual) - - -def test_get_raw_data(valid_dw): - data = valid_dw.get_raw_data( - duration=('2023-05-01', '2023-05-02'), - realm='SUPREMM', - fields=( - 'CPU User', - 'Nodes', - 'Wall Time', - 'Wait Time', - 'Requested Wall Time', - 'Total memory used', - 'Mount point "home" data written', - 'Mount point "scratch" data written', + if 'GENERATE_DATA_FILES' in os.environ: # pragma: no cover + try: + os.mkdir(DATA_DIR) + except FileExistsError: + pass + actual.to_csv(DATA_DIR + '/' + data_file) + else: + expected = pandas.read_csv( + DATA_DIR + '/' + data_file, + dtype=dtype, + index_col=index_col, + keep_default_na=False, + na_values=[''], + ).fillna(numpy.nan) + expected.columns = expected.columns.astype('string') + expected.columns.name = columns_name + if index_col == 'Time': + expected.index = pandas.to_datetime(expected.index) + assert expected.equals(actual) + + +@pytest.mark.parametrize( + 'additional_params, number, csv_title', + [ + ( + {}, + '54747', + 'raw-data-every-1000-no-fields-no-filters.csv', ), - filters={ - 'Resource': [ - 'STAMPEDE2 TACC', - 'Bridges 2 RM', - ], - }, - ).iloc[::1000] + ( + { + 'fields': + ( + 'Local Job Id', + 'Quality of Service', + 'GPUs', + 'Start Time', + 'Department', + ), + 'filters': + { + 'Resource': + [ + 'mortorq', + 'frearson', + ], + }, + }, + '33345', + 'raw-data-every-1000-with-fields-and-filters.csv', + ), + ], +) +def test_get_raw_data(valid_dw, capsys, additional_params, number, csv_title): + defult_params = { + 'duration': ('2016-01-01', '2016-12-31'), + 'realm': 'Jobs', + 'show_progress': True, + } + params = {**defult_params, **additional_params} + data = valid_dw.get_raw_data(**params).iloc[::1000] data.index = data.index.astype('string') __assert_dfs_equal( - 'machine-learning-notebook-example-every-1000.csv', + csv_title, data, dtype='string', index_col=0, ) + assert 'Got ' + number + ' rows...DONE' in capsys.readouterr().out def __assert_descriptor_dfs_equal(data_file, actual): @@ -72,6 +106,7 @@ def __assert_descriptor_dfs_equal(data_file, actual): def test_describe_realms(valid_dw): + __assert_descriptor_dfs_equal( 'realms.csv', valid_dw.describe_realms(), @@ -94,8 +129,8 @@ def test_describe_dimensions(valid_dw): def test_get_filter_values(valid_dw): __assert_descriptor_dfs_equal( - 'jobs-fieldofscience-filter-values.csv', - valid_dw.get_filter_values('Jobs', 'Field of Science'), + 'jobs-pi-group-filter-values.csv', + valid_dw.get_filter_values('Jobs', 'PI Group'), ) @@ -103,9 +138,68 @@ def test_get_data_filter_user(valid_dw): # Make sure the filter validation works for a user whose list position is # greater than 10000 — this will raise an exception if it doesn't work. valid_dw.get_data( - duration=('2023-01-01', '2023-01-01'), + duration=('2016-01-01', '2017-12-31'), realm='Jobs', metric='CPU Hours: Total', dataset_type='aggregate', filters={'User': '10332'}, ) + + +@pytest.mark.parametrize( + 'aggregation_unit,data_file', + [ + ('Month', 'jobs-2016-2017-month.csv'), + ('Quarter', 'jobs-2016-2017-quarters.csv'), + ('Year', 'jobs-2016-2017-years.csv'), + ], + ids=('month', 'quarter', 'year'), +) +def test_get_data(valid_dw, aggregation_unit, data_file): + data = valid_dw.get_data( + duration=('2016-01-01', '2017-12-31'), + realm='Jobs', + metric='CPU Hours: Total', + aggregation_unit=aggregation_unit, + ) + __assert_dfs_equal( + data_file, + data, + index_col='Time', + columns_name='Metric', + dtype={'CPU Hours: Total': 'Float64'}, + ) + + +def test_get_aggregation_units(valid_dw): + expected_agg_units = ('Auto', 'Day', 'Month', 'Quarter', 'Year') + actual_agg_units = valid_dw.get_aggregation_units() + assert expected_agg_units == actual_agg_units + + +def test_get_durations(valid_dw): + expected_durations = [ + 'Yesterday', + '7 day', + '30 day', + '90 day', + 'Month to date', + 'Previous month', + 'Quarter to date', + 'Previous quarter', + 'Year to date', + 'Previous year', + '1 year', + '2 year', + '3 year', + '5 year', + '10 year', + ] + today_date = date.today() + current_year = today_date.year + for count in range(0, 7): + year = current_year - count + year = str(year) + expected_durations.append(year) + actual_durations = list(valid_dw.get_durations()) + assert expected_durations == actual_durations diff --git a/tests/unit/test_datawarehouse_unit.py b/tests/unit/test_datawarehouse_unit.py index 7af2d989..9627e0f1 100644 --- a/tests/unit/test_datawarehouse_unit.py +++ b/tests/unit/test_datawarehouse_unit.py @@ -4,13 +4,17 @@ from xdmod_data.warehouse import DataWarehouse -VALID_XDMOD_URL = 'https://xdmod.access-ci.org' +VALID_XDMOD_HOST = os.environ['XDMOD_HOST'] INVALID_STR = 'asdlkfjsdlkfisdjkfjd' @pytest.fixture(scope='module', autouse=True) def set_environ(): - token = os.environ['XDMOD_API_TOKEN'] if 'XDMOD_API_TOKEN' in os.environ else '' + token = ( + os.environ['XDMOD_API_TOKEN'] + if 'XDMOD_API_TOKEN' in os.environ + else '' + ) os.environ['XDMOD_API_TOKEN'] = INVALID_STR yield os.environ['XDMOD_API_TOKEN'] = token @@ -28,23 +32,31 @@ def test___init___KeyError(): KeyError, match='`XDMOD_API_TOKEN` environment variable has not been set.', ): - DataWarehouse(VALID_XDMOD_URL) + DataWarehouse(VALID_XDMOD_HOST) os.environ['XDMOD_API_TOKEN'] = token def test___enter___RuntimeError_xdmod_host_malformed(): with pytest.raises( - requests.exceptions.InvalidURL, - match=r'Invalid URL \'.*\': No host supplied' + ( + requests.exceptions.InvalidURL, + requests.exceptions.MissingSchema, + ), + match=( + r'(Invalid URL \'.*\': No host supplied|' + + r'Invalid URL \'https:\?Bearer=' + INVALID_STR + "': " + + r'No schema supplied. Perhaps you meant http://https:\?Bearer=' + + INVALID_STR + r'\?)' + ), ): - with DataWarehouse('https://'): + with DataWarehouse('https://'): # pragma: no cover pass def test___enter___RuntimeError_xdmod_host_unresolved(): invalid_host = 'https://' + INVALID_STR + '.xdmod.org' with pytest.raises(Exception): - with DataWarehouse(invalid_host): + with DataWarehouse(invalid_host): # pragma: no cover pass @@ -52,9 +64,9 @@ def test___enter___RuntimeError_xdmod_host_unsupported_protocol(): invalid_host = INVALID_STR + '://' + INVALID_STR with pytest.raises( requests.exceptions.InvalidSchema, - match='No connection adapters were found for \'' + invalid_host + match="No connection adapters were found for '" + invalid_host, ): - with DataWarehouse(invalid_host): + with DataWarehouse(invalid_host): # pragma no cover pass @@ -62,7 +74,12 @@ def test___enter___RuntimeError_401(): with pytest.raises( RuntimeError, match='Error 401: Make sure XDMOD_API_TOKEN is set' - + ' to a valid API token.' + + ' to a valid API token.', ): - with DataWarehouse(VALID_XDMOD_URL) as dw: + with DataWarehouse(VALID_XDMOD_HOST) as dw: dw.describe_realms() + + +def test_exit_without_enter(): + dw = DataWarehouse(VALID_XDMOD_HOST) + dw.__exit__(None, None, None) diff --git a/xdmod_data/__version__.py b/xdmod_data/__version__.py index 10f5e82c..475e210b 100644 --- a/xdmod_data/__version__.py +++ b/xdmod_data/__version__.py @@ -1,2 +1,2 @@ __title__ = 'xdmod-data' -__version__ = '1.0.1-beta.1' +__version__ = '1.1.0-beta.1' diff --git a/xdmod_data/_descriptors.py b/xdmod_data/_descriptors.py index 6387ba68..5826737e 100644 --- a/xdmod_data/_descriptors.py +++ b/xdmod_data/_descriptors.py @@ -19,15 +19,15 @@ def __request_aggregate(self): '/controllers/metric_explorer.php', {'operation': 'get_dw_descripter'}, ) - if response['totalCount'] != 1: + if response['totalCount'] != 1: # pragma: no cover raise RuntimeError( - 'Descriptor received with unexpected structure.' + 'Descriptor received with unexpected structure.', ) return self.__deserialize_aggregate(response['data'][0]['realms']) def __request_raw(self): response = self.__http_requester._request_json( - '/rest/v1/warehouse/export/realms' + '/rest/v1/warehouse/export/realms', ) return self.__deserialize_raw(response['data']) diff --git a/xdmod_data/_http_requester.py b/xdmod_data/_http_requester.py index d90e3af7..69916bbe 100644 --- a/xdmod_data/_http_requester.py +++ b/xdmod_data/_http_requester.py @@ -1,5 +1,6 @@ import json import os +import re import requests from urllib.parse import urlencode import xdmod_data._validator as _validator @@ -10,12 +11,13 @@ class _HttpRequester: def __init__(self, xdmod_host): self.__in_runtime_context = False _validator._assert_str('xdmod_host', xdmod_host) + xdmod_host = re.sub('/+$', '', xdmod_host) self.__xdmod_host = xdmod_host try: self.__api_token = os.environ['XDMOD_API_TOKEN'] except KeyError: raise KeyError( - '`XDMOD_API_TOKEN` environment variable has not been set.' + '`XDMOD_API_TOKEN` environment variable has not been set.', ) from None self.__headers = { 'Authorization': 'Bearer ' + self.__api_token, @@ -42,24 +44,55 @@ def _request_data(self, params): def _request_raw_data(self, params): url_params = self.__get_raw_data_url_params(params) + # Once XDMoD 10.5 is no longer supported, there will be no need to call + # __get_raw_data_limit(), and the if/else statement below will not be + # necessary — only the body of the 'if' branch will be needed. limit = self.__get_raw_data_limit() data = [] - num_rows = limit - offset = 0 - while num_rows == limit: - response = self._request_json( - path='/rest/v1/warehouse/raw-data?' + url_params - + '&offset=' + str(offset) + if limit == 'NA': + response_iter_lines = self.__request( + path='/rest/v1/warehouse/raw-data?' + url_params, + post_fields=None, + stream=True, ) - partial_data = response['data'] - data += partial_data + i = 0 + for line in response_iter_lines: + line_text = line.decode('utf-8').replace('\x1e', '') + line_json = json.loads(line_text) + if i == 0: + response = {'fields': line_json} + else: + data.append(line_json) + if params['show_progress']: + progress_msg = ( + 'Got ' + str(i) + ' row' + ('' if i == 1 else 's') + + '...' + ) + print(progress_msg, end='\r') + i += 1 if params['show_progress']: - progress_msg = 'Got ' + str(len(data)) + ' rows...' - print(progress_msg, end='\r') - num_rows = len(partial_data) - offset += limit - if params['show_progress']: - print(progress_msg + 'DONE') + print(progress_msg + 'DONE') + else: + num_rows = limit + offset = 0 + while num_rows == limit: + response = self._request_json( + path='/rest/v1/warehouse/raw-data?' + url_params + + '&offset=' + str(offset), + ) + partial_data = response['data'] + data += partial_data + if params['show_progress']: + progress_msg = ( + 'Got ' + str(len(data)) + ' row' + + ('' if len(data) == 1 else 's') + + '...' + ) + print(progress_msg, end='\r') + num_rows = len(partial_data) + offset += limit + if params['show_progress']: + print(progress_msg + 'DONE') return (data, response['fields']) def _request_filter_values(self, realm_id, dimension_id): @@ -90,13 +123,13 @@ def _request_json(self, path, post_fields=None): def __assert_connection_to_xdmod_host(self): try: self.__request() - except RuntimeError as e: + except RuntimeError as e: # pragma: no cover raise RuntimeError( - 'Could not connect to xdmod_host \'' + self.__xdmod_host - + '\': ' + str(e) + "Could not connect to xdmod_host '" + self.__xdmod_host + + "': " + str(e), ) from None - def __request(self, path='', post_fields=None): + def __request(self, path='', post_fields=None, stream=False): _validator._assert_runtime_context(self.__in_runtime_context) url = self.__xdmod_host + path if post_fields: @@ -115,16 +148,19 @@ def __request(self, path='', post_fields=None): try: response_json = json.loads(response.text) msg = ': ' + response_json['message'] - except json.JSONDecodeError: + except json.JSONDecodeError: # pragma: no cover pass if response.status_code == 401: msg = ( ': Make sure XDMOD_API_TOKEN is set to a valid API token.' ) raise RuntimeError( - 'Error ' + str(response.status_code) + msg + 'Error ' + str(response.status_code) + msg, ) from None - return response.text + if stream: + return response.iter_lines() + else: + return response.text def __get_data_post_fields(self, params): post_fields = { @@ -140,7 +176,7 @@ def __get_data_post_fields(self, params): } for dimension in params['filters']: post_fields[dimension + '_filter'] = ','.join( - params['filters'][dimension] + params['filters'][dimension], ) return post_fields @@ -155,12 +191,22 @@ def __get_raw_data_url_params(self, params): if (params['filters']): for dimension in params['filters']: results['filters[' + dimension + ']'] = ','.join( - params['filters'][dimension] + params['filters'][dimension], ) return urlencode(results) + # Once XDMoD 10.5 is no longer supported, + # there will be no need for this method. def __get_raw_data_limit(self): if self.__raw_data_limit is None: - response = self._request_json('/rest/v1/warehouse/raw-data/limit') - self.__raw_data_limit = int(response['data']) + try: + response = self._request_json( + '/rest/v1/warehouse/raw-data/limit', + ) + self.__raw_data_limit = int(response['data']) + except RuntimeError as e: + if '404' in str(e): + self.__raw_data_limit = 'NA' + else: # pragma: no cover + raise return self.__raw_data_limit diff --git a/xdmod_data/_response_processor.py b/xdmod_data/_response_processor.py index ba06f925..e015634f 100644 --- a/xdmod_data/_response_processor.py +++ b/xdmod_data/_response_processor.py @@ -18,7 +18,7 @@ def _process_get_data_response(dw, params, response): csv_data = csv.reader(response.splitlines()) if params['dataset_type'] == 'timeseries': return __parse_timeseries_csv_data(dw, params, csv_data) - elif params['dataset_type'] == 'aggregate': + else: return __parse_aggregate_csv_data(params, csv_data) @@ -55,10 +55,7 @@ def __parse_timeseries_dimension_values(labels): dimension_values = [] for label in labels: match = label_re.match(label) - if match: - dimension_values.append(html.unescape(match.group(1))) - else: - dimension_values.append(html.unescape(label)) + dimension_values.append(html.unescape(match.group(1))) return dimension_values @@ -75,9 +72,9 @@ def __parse_timeseries_date_string(date_string): # Match YYYY Q# elif re.match(r'^[0-9]{4} Q[0-9]$', date_string): (date_string, format_) = __parse_quarter_date_string(date_string) - else: + else: # pragma: no cover raise Exception( - 'Unsupported date specification ' + date_string + '.' + 'Unsupported date specification ' + date_string + '.', ) return datetime.strptime(date_string, format_) @@ -134,10 +131,10 @@ def __parse_quarter_date_string(date_string): month = '07' elif quarter == 'Q4': month = '10' - else: + else: # pragma: no cover raise Exception( 'Unsupported date quarter specification ' - + date_string + '.' + + date_string + '.', ) date_string = year + '-' + month + '-01' format_ = '%Y-%m-%d' diff --git a/xdmod_data/_validator.py b/xdmod_data/_validator.py index d80f399a..646aa7f1 100644 --- a/xdmod_data/_validator.py +++ b/xdmod_data/_validator.py @@ -10,7 +10,7 @@ def _assert_runtime_context(in_runtime_context): raise RuntimeError( 'Method is being called outside of the runtime context.' + ' Make sure this method is only called within the body' - + ' of a `with` statement.' + + ' of a `with` statement.', ) @@ -94,7 +94,7 @@ def _get_durations(): this_year = date.today().year six_years_ago = this_year - 6 last_seven_years = tuple( - map(str, reversed(range(six_years_ago, this_year + 1))) + map(str, reversed(range(six_years_ago, this_year + 1))), ) return ( ( @@ -156,7 +156,7 @@ def __validate_duration(duration): except (TypeError, ValueError) as error: raise type(error)( '`duration` must be a string or an object' - + ' with 2 items.' + + ' with 2 items.', ) from None return (start_date, end_date) @@ -194,7 +194,7 @@ def __validate_filters(data_warehouse, descriptors, realm, filters): except TypeError: raise TypeError( '`filters` must be a mapping whose keys are strings and whose' - + ' values are strings or sequences of strings.' + + ' values are strings or sequences of strings.', ) from None @@ -210,8 +210,8 @@ def __find_str_in_sequence(value, sequence, label): if transformed_valid_value == transformed_value: return valid_value raise KeyError( - 'Invalid value for `' + label + '`: \'' + value + '\'' - + '. Valid values are: \'' + '\', \''.join(sequence) + '\'.' + 'Invalid value for `' + label + "`: '" + value + "'" + + ". Valid values are: '" + "', '".join(sequence) + "'.", ) from None @@ -225,7 +225,7 @@ def __validate_raw_fields(data_warehouse, realm, fields): return results except TypeError: raise TypeError( - '`fields` must be a sequence of strings.' + '`fields` must be a sequence of strings.', ) from None @@ -235,7 +235,7 @@ def __find_id_in_descriptor(descriptor, name, value): if id_ == value or descriptor[id_]['label'] == value: return id_ raise KeyError( - name.capitalize() + ' \'' + value + '\' not found.' + name.capitalize() + " '" + value + "' not found.", ) @@ -254,10 +254,10 @@ def __get_dates_from_duration(duration): last_month = today + timedelta(days=-30) last_quarter = today + timedelta(days=-90) this_month_start = date(today.year, today.month, 1) - if today.month == 1: + if today.month == 1: # pragma: no cover last_full_month_start_year = today.year - 1 last_full_month_start_month = 12 - else: + else: # pragma: no cover last_full_month_start_year = today.year last_full_month_start_month = today.month - 1 last_full_month_start = date( @@ -271,9 +271,9 @@ def __get_dates_from_duration(duration): ((today.month - 1) // 3) * 3 + 1, 1, ) - if today.month < 4: + if today.month < 4: # pragma: no cover last_quarter_start_year = today.year - 1 - else: + else: # pragma: no cover last_quarter_start_year = today.year last_quarter_start = date( last_quarter_start_year, @@ -321,7 +321,7 @@ def __find_value_in_df(label, df, value): elif value in df['label'].values: return df.index[df['label'] == value].tolist()[0] else: - raise KeyError(label + ' \'' + value + '\' not found.') + raise KeyError(label + " '" + value + "' not found.") def __lowercase_and_remove_spaces(value): @@ -340,7 +340,7 @@ def __date_add_years(old_date, year_delta): try: new_date = date(new_date_year, old_date.month, new_date_day) keep_going = False - except ValueError: + except ValueError: # pragma: no cover new_date_day -= 1 days_above += 1 return new_date + timedelta(days=days_above) diff --git a/xdmod_data/themes.py b/xdmod_data/themes.py index 14d4bf29..ebf41167 100644 --- a/xdmod_data/themes.py +++ b/xdmod_data/themes.py @@ -1,76 +1,76 @@ import plotly.graph_objects as go import plotly.io as pio -pio.templates["timeseries"] = go.layout.Template( +pio.templates['timeseries'] = go.layout.Template( layout={ 'hoverlabel': { - 'bgcolor': 'white' + 'bgcolor': 'white', }, 'plot_bgcolor': 'white', 'colorway': [ - "#1199FF", - "#DB4230", - "#4E665D", - "#F4A221", - "#66FF00", - "#33ABAB", - "#A88D95", - "#789ABC", - "#FF99CC", - "#00CCFF", - "#FFBC71", - "#A57E81", - "#8D4DFF", - "#FF6666", - "#CC99FF", - "#2F7ED8", - "#0D233A", - "#8BBC21", - "#910000", - "#1AADCE", - "#492970", - "#F28F43", - "#77A1E5", - "#3366FF", - "#FF6600", - "#808000", - "#CC99FF", - "#008080", - "#CC6600", - "#9999FF", - "#99FF99", - "#969696", - "#FF00FF", - "#FFCC00", - "#666699", - "#00FFFF", - "#00CCFF", - "#993366", - "#3AAAAA", - "#C0C0C0", - "#FF99CC", - "#FFCC99", - "#CCFFCC", - "#CCFFFF", - "#99CCFF", - "#339966", - "#FF9966", - "#69BBED", - "#33FF33", - "#6666FF", - "#FF66FF", - "#99ABAB", - "#AB8722", - "#AB6565", - "#990099", - "#999900", - "#CC3300", - "#669999", - "#993333", - "#339966", - "#C42525", - "#A6C96A", - "#111111", + '#1199FF', + '#DB4230', + '#4E665D', + '#F4A221', + '#66FF00', + '#33ABAB', + '#A88D95', + '#789ABC', + '#FF99CC', + '#00CCFF', + '#FFBC71', + '#A57E81', + '#8D4DFF', + '#FF6666', + '#CC99FF', + '#2F7ED8', + '#0D233A', + '#8BBC21', + '#910000', + '#1AADCE', + '#492970', + '#F28F43', + '#77A1E5', + '#3366FF', + '#FF6600', + '#808000', + '#CC99FF', + '#008080', + '#CC6600', + '#9999FF', + '#99FF99', + '#969696', + '#FF00FF', + '#FFCC00', + '#666699', + '#00FFFF', + '#00CCFF', + '#993366', + '#3AAAAA', + '#C0C0C0', + '#FF99CC', + '#FFCC99', + '#CCFFCC', + '#CCFFFF', + '#99CCFF', + '#339966', + '#FF9966', + '#69BBED', + '#33FF33', + '#6666FF', + '#FF66FF', + '#99ABAB', + '#AB8722', + '#AB6565', + '#990099', + '#999900', + '#CC3300', + '#669999', + '#993333', + '#339966', + '#C42525', + '#A6C96A', + '#111111', ], 'xaxis': { 'titlefont': { @@ -88,7 +88,7 @@ 'zerolinecolor': '#c0cfe1', 'showline': True, 'zerolinewidth': 4, - 'tickformat': "%Y-%m-%d", + 'tickformat': '%Y-%m-%d', }, 'yaxis': { 'titlefont': {