Docker compose

.github/PULL_REQUEST_TEMPLATE.md

@@ -21,6 +21,7 @@
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
+- [ ] Release preparation
 
 ## Checklist:
 <!--- Go over all the following points and make sure they have all been completed -->

.github/workflows/docker-compose.yaml

@@ -0,0 +1,13 @@
+services:
+    xdmod10.5:
+        container_name: xdmod10.5
+        image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
+        tty: true
+        ports: 
+          - "8081:443"      
+    xdmod-main-dev:
+        container_name: xdmod-main-dev
+        image: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
+        tty: true
+        ports:
+          - "8082:443"

.github/workflows/python-package.yml

@@ -16,7 +16,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: [
+          "3.12",
+          "3.11",
+          "3.10",
+          "3.9",
+          "3.8",
+        ]
 
     steps:
     - uses: actions/checkout@v3
@@ -27,15 +33,69 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest pandas
+        python -m pip install flake8 flake8-commas flake8-quotes pandas python-dotenv pytest coverage
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         python -m pip install -e .
     - name: Lint with flake8
       run: |
         # stop the build if there are linter errors (ignore complexity metrics)
-        flake8 . --count --max-line-length=127 --max-complexity=1000 --show-source --statistics --exclude __init__.py
-        # output complexity metrics
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude __init__.py
-    - name: Test with pytest
+        flake8 . --max-complexity=10 --show-source --exclude __init__.py
+    - name: Spin up containers for testing XDMoD webservers
+      uses: hoverkraft-tech/compose-action@v2.0.1
+      with: 
+        compose-file: .github/workflows/docker-compose.yaml
+    - name: Install yq 
       run: |
-        pytest tests/unit
+        sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
+    - name: Test against each supported XDMoD portal version
+      run: |
+        # Create a file with 10,000 users so that we can test filters that have more than 10,000 values
+        for i in {1..10000}; do
+          # sets the unix at the end of quarter 2
+          unix=$((1498866840+$i*2))
+          # each user is added in the file later to be injested
+          echo "$i|2269212|frearson|black|banana-cream|caste|caste|45580|user$i|379330|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$unix)|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+1)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+2)))|$(date --utc +'%Y-%m-%dT%H:%M:%S' -d @$(($unix+3)))|2-23:24:40|0:0|COMPLETED|1|16|16|62.50Gn|cpu=16,mem=62.50G,node=1|billing=16,cpu=16,mem=62.50G,node=1|3-00:00:00|cpn-m25-02-01|310f6011b278ebb15df2d199e0119de78d261d8c478a203abc97dc3dda7702ae" >> 10000users.log
+        done
+        #run test
+        coverage erase
+        declare -a portal_versions=$(yq '.services | keys[]' .github/workflows/docker-compose.yaml)
+        for version in $portal_versions; do
+          echo "starting up $version container"
+          # Generate OpenSSL Key
+          docker exec $version sh -c 'openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key'
+          # Generate Certificate 
+          docker exec $version /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd/CN=localhost" -addext 'subjectAltName=DNS:localhost'
+          # Copy Certificate from container
+          docker cp $version:/etc/pki/tls/certs/localhost.crt $version.crt 
+          if [ "$version" = "xdmod-main-dev" ]; then
+            # Install the latest development version of xdmod, 
+            # these steps come from xdmod CircleCI configuration file
+            docker exec xdmod-main-dev bash -c "dnf module -y reset php && dnf module -y enable php:7.4"
+            docker exec xdmod-main-dev bash -c "dnf install -y php libzip-devel php-pear php-devel"
+            docker exec xdmod-main-dev bash -c "yes '' | pecl install mongodb || true"
+            docker exec xdmod-main-dev bash -c "git clone --depth=1 https://github.com/ubccr/xdmod.git /root/xdmod"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "composer install"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "~/bin/buildrpm xdmod"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "XDMOD_TEST_MODE=upgrade ./tests/ci/bootstrap.sh"
+            docker exec -w /root/xdmod xdmod-main-dev bash -c "./tests/ci/validate.sh"
+          else
+            # Start up web server in container
+            docker exec $version /root/bin/services start
+          fi
+          # Prepare to use certificate for curl and python requests
+          export CURL_CA_BUNDLE="$version.crt"
+          ports=$(yq ".services.\"$version\".ports[0]" .github/workflows/docker-compose.yaml)
+          port=$(echo $ports | cut -d ':' -f 1)
+          export XDMOD_HOST="https://localhost:$port"
+          token=$(curl -X POST -c xdmod.cookie -d "username=normaluser&password=normaluser" $XDMOD_HOST/rest/auth/login | jq -r '.results.token')
+          echo -n 'XDMOD_API_TOKEN=' > ~/.xdmod-data-token
+          curl -X POST -b xdmod.cookie $XDMOD_HOST/rest/users/current/api/token?token=$token | jq -r '.data.token' >> ~/.xdmod-data-token
+          docker cp 10000users.log $version:10000users.log
+          docker exec $version xdmod-shredder -r frearson -f slurm -i 10000users.log
+          date=$(date --utc +%Y-%m-%d)
+          docker exec $version xdmod-ingestor --ingest
+          docker exec $version xdmod-ingestor --aggregate=job --last-modified-start-date $date
+          export XDMOD_VERSION="$version"
+          coverage run --branch --append -m pytest -vvs -o log_cli=true tests/
+        done
+        coverage report -m --fail-under=100

CHANGELOG.md

@@ -1,10 +1,23 @@
 # xdmod-data Changelog
 
 ## Main development branch
+- Implement 100% test coverage ([\#27](https://github.com/ubccr/xdmod-data/pull/27)).
+- Update Flake8 rules ([\#28](https://github.com/ubccr/xdmod-data/pull/28)).
+
+## v1.0.1 (2024-06-XX)
+This release has bug fixes, performance improvements, and updates for compatibility, tests, and documentation.
+- Add borders to README images ([\#12](https://github.com/ubccr/xdmod-data/pull/12)).
+- Create `PULL_REQUEST_TEMPLATE.md` ([\#13](https://github.com/ubccr/xdmod-data/pull/13)).
+- Add types of changes to pull request template ([\#15](https://github.com/ubccr/xdmod-data/pull/15)).
+- Add Changelog ([\#17](https://github.com/ubccr/xdmod-data/pull/17)).
 - Update tests and testing instructions ([\#14](https://github.com/ubccr/xdmod-data/pull/14)).
 - Remove limit on number of results returned from `get_filter_values()` ([\#21](https://github.com/ubccr/xdmod-data/pull/21)).
-- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-notebooks/pull/22)).
+- Add citation for the DAF paper ([\#23](https://github.com/ubccr/xdmod-data/pull/23)).
+- Add a "Feedback / Feature Requests" section to the README ([\#22](https://github.com/ubccr/xdmod-data/pull/22)).
 - Improve performance of validation of filters and raw fields ([\#18](https://github.com/ubccr/xdmod-data/pull/18)).
+- Fix bug with trailing slashes in `xdmod_host` ([\#24](https://github.com/ubccr/xdmod-data/pull/24)).
+- Use streaming for raw data requests ([\#19](https://github.com/ubccr/xdmod-data/pull/19)).
+- Specify minimum version numbers for dependencies ([\#25](https://github.com/ubccr/xdmod-data/pull/25)). 
 
 ## v1.0.0 (2023-07-21)
 - Initial release.

README.md

@@ -1,12 +1,20 @@
 # xdmod-data
-As part of the [XDMoD](https://open.xdmod.org) Data Analytics Framework, this Python package provides API access to the data warehouse of an instance of XDMoD version ≥10.5.
+As part of the Data Analytics Framework for [XDMoD](https://open.xdmod.org), this Python package provides API access to the data warehouse of instances of Open XDMoD.
 
 The package can be installed from PyPI via `pip install xdmod-data`.
 
 It has dependencies on [NumPy](https://pypi.org/project/numpy/), [Pandas](https://pypi.org/project/pandas/), [Plotly](https://pypi.org/project/plotly/), and [Requests](https://pypi.org/project/requests/).
 
 Example usage is documented through Jupyter notebooks in the [xdmod-notebooks](https://github.com/ubccr/xdmod-notebooks) repository.
 
+## Compatibility with Open XDMoD
+Specific versions of this `xdmod-data` API are compatible with specific versions of Open XDMoD as indicated in the table below.
+
+| `xdmod-data` version | Open XDMoD versions |
+| -------------------- | ------------------- |
+| 1.0.1                | 11.0.x, 10.5.x      |
+| 1.0.0                | 10.5.x              |
+
 ## API Token Access
 Use of the Data Analytics Framework requires an API token. To obtain an API token, follow the steps below to obtain an API token from the XDMoD portal.
 
@@ -55,7 +63,13 @@ For support, please see [this page](https://open.xdmod.org/support.html). If you
 ## License
 `xdmod-data` is released under the GNU Lesser General Public License ("LGPL") Version 3.0. See the [LICENSE](LICENSE) file for details.
 
-## Reference
+## References
+
+When referencing the Data Analytics Framework for XDMoD, please cite the following publication:
+
+> Weeden, A., White, J.P., DeLeon, R.L., Rathsam, R., Simakov, N.A., Saeli, C., and Furlani, T.R. The Data Analytics Framework for XDMoD. _SN COMPUT. SCI._ 5, 462 (2024). https://doi.org/10.1007/s42979-024-02789-2
+
 When referencing XDMoD, please cite the following publication:
 
-> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:10.1109/MCSE.2015.68
+> Jeffrey T. Palmer, Steven M. Gallo, Thomas R. Furlani, Matthew D. Jones, Robert L. DeLeon, Joseph P. White, Nikolay Simakov, Abani K. Patra, Jeanette Sperhac, Thomas Yearke, Ryan Rathsam, Martins Innus, Cynthia D. Cornelius, James C. Browne, William L. Barth, Richard T. Evans, "Open XDMoD: A Tool for the Comprehensive Management of High-Performance Computing Resources", *Computing in Science & Engineering*, Vol 17, Issue 4, 2015, pp. 52-62. DOI:[10.1109/MCSE.2015.68](https://doi.org/10.1109/MCSE.2015.68)
+

docs/developing.md

@@ -1,53 +1,114 @@
-# Testing the code
+# Instructions for developers
+
+## Testing the code
 1. Start up a virtual environment, e.g.:
+    ```                                            
+    python3 -m venv ~/xdmod-data-test-env                                             
+    source ~/xdmod-data-test-env/bin/activate
     ```
-    $ env_dir=~/xdmod-data-test-env                                            
-    $ python3 -m venv ${env_dir}                                                
-    $ source ${env_dir}/bin/activate
-    ```
+    Your command prompt should now start with `(xdmod-data-test-env)`.
 1. Install your local development copy of `xdmod-data` in editable mode:
     ```
-    (env) $ python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
+    python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
     ```
-1. Install `python-dotenv` and `pytest`:
+1. Install `python-dotenv`, `pytest`, and `coverage`:
     ```
-    (env) $ python3 -m pip install --upgrade python-dotenv pytest
+    python3 -m pip install --upgrade python-dotenv pytest coverage
     ```
 1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
-1. With an [https://xdmod.access-ci.org](ACCESS XDMoD) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
+1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
 1. Add the following line to the file `~/.xdmod-data-token`, replacing `<token>` with your token.
     ```
     XDMOD_API_TOKEN=<token>
     ```
 1. Change directories to your local development copy of `xdmod-data`.
-1. Run the following command:
+1. Run the following command and make sure all the tests pass:
     ```
-    (env) $ pytest -vvs -o log_cli=true tests/ 
+    coverage run -m pytest -vvs -o log_cli=true tests/
     ```
-
-# Developing a new version
-1. Make sure the version number is updated in `xdmod_data/__version__.py`.
-1. Create a Pull Request for the new version.
-
-# Distributing the new version to PyPI
-After the Pull Request is approved, follow these steps in the `xdmod-data` root directory:
-1. Start up a virtual environment, e.g.:
+1. Run the following command and make sure the code is 100% covered by tests:
+    ```
+    coverage report -m
     ```
-    $ env_dir=~/xdmod-data-build-env
-    $ python3 -m venv ${env_dir}
-    $ source ${env_dir}/bin/activate
+1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
     ```
-1. Make sure the required packages are installed:
+    python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0
     ```
-    (env) $ python3 -m pip install --upgrade pip build setuptools twine
+1. Run the following command again and make sure all the tests pass (Deprecation warnings in `urllib3` are Ok).
     ```
-1. Build the built distribution:
+    pytest -vvs -o log_cli=true tests/
+    ```
+## Linting the code
+1. Start up a virtual environment, e.g.:
+    ```                                            
+    python3 -m venv ~/xdmod-data-test-env                                             
+    source ~/xdmod-data-test-env/bin/activate
     ```
-    (env) $ python3 -m build --wheel
+    Your command prompt should now start with `(xdmod-data-test-env)`.
+1. Install Flake8 and additional packages
     ```
-1. Upload the built distribution to PyPI, e.g., for version 1.0.0-beta1:
+    python3 -m pip install flake8 flake8-quotes flake8-commas
     ```
-    (env) $ version=1.0.0b1
-    (env) $ twine upload dist/xdmod_data-${version}-py3-none-any.whl
+1. Run Flake8
     ```
-    Enter your PyPI username and password.
+    flake8 --max-complexity=10 --show-source --exclude __init__.py /path/to/your/xdmod-data
+    ```
+## Releasing a new version
+1. Make a new branch of `xdmod-data` and:
+
+    1. Make sure the version number is updated in `xdmod_data/__version__.py`.
+    1. Update the Open XDMoD compatibility matrix in the `README.md`.
+    1. Update `CHANGELOG.md` to change "Main development branch" to, e.g., `v1.0.1 (2024-06-XX)`.
+    1. Create a Pull Request for the new version.
+
+1. After the Pull Request is approved (but not merged yet), follow these steps in a cloned copy of the branch:
+    1. Start up a virtual environment, e.g.:
+        ```
+        python3 -m venv ~/xdmod-data-build-env
+        source ~/xdmod-data-build-env/bin/activate
+        ```
+        Your command prompt should now start with `(xdmod-data-build-env)`.
+    1. Make sure the required packages are installed:
+        ```
+        python3 -m pip install --upgrade pip build setuptools twine
+        ```
+    1. Build the built distribution:
+        ```
+        python3 -m build --wheel
+        ```
+    1. Validate `setup.cfg`, e.g., for version 1.0.0-beta1:
+        ```
+        version=1.0.0b1
+        twine check dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Make sure you receive `PASSED`.
+    1. Upload the built distribution to TestPyPI:
+        ```
+        twine upload --repository testpypi dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Enter your TestPyPI API token.
+    1. Go to https://testpypi.org/project/xdmod-data and confirm that everything looks right.
+    1. Upload the built distribution to PyPI:
+        ```
+        twine upload dist/xdmod_data-${version}-py3-none-any.whl
+        ```
+        Enter your PyPI username and password.
+    1. Go to https://pypi.org/project/xdmod-data and confirm the new version is the latest release.
+1. Update the date of the release in `CHANGELOG.md` and commit it to the Pull Request.
+1. Merge the Pull Request.
+
+1. Go to [create a new release on GitHub](https://github.com/ubccr/xdmod-data/releases/new) and:
+    1. Click `Choose a tag`.
+    1. Type in a tag similar to `v1.0.0` and choose `Create new tag`.
+    1. Make the release title the same as the tag name.
+    1. Where it says `Describe this release` paste in the contents of `CHANGELOG.md`.
+    1. Where it says `Attach binaries` attach the built distribution that was uploaded to PyPI.
+    1. Click `Publish release`.
+
+## After release
+
+1. Make a new branch of `xdmod-data` and:
+
+    1. Make sure the version number is updated in `xdmod_data/__version__.py` to a beta release of the next version, e.g., `1.0.1-beta.1`.
+    1. Update `CHANGELOG.md` to add a section at the top called `Main development branch`.
+    1. Create a Pull Request for the new version.

setup.cfg

@@ -4,17 +4,17 @@ version = attr: xdmod_data.__version__
 author = Aaron Weeden, Joseph P. White
 maintainer = Aaron Weeden, Joseph P. White
 description = Python package for XDMoD data access
-long_description = file: README.md
+long_description = See [the README](https://github.com/ubccr/xdmod-data?tab=readme-ov-file#xdmod-data) for instructions on use.
 long_description_content_type = text/markdown
 license_files = LICENSE
 project_urls =
     Source Code = https://github.com/ubccr/xdmod-data
 
 [options]
 packages = find:
-python_requires = >=3
+python_requires = >=3.8
 install_requires =
-    numpy
-    pandas
-    plotly
-    requests
+    numpy >= 1.23.0
+    pandas >= 1.5.0
+    plotly >= 5.8.0
+    requests >= 2.19.0