Add github workflow to validate samples

.github/workflows/clean-docs.yml

@@ -14,9 +14,9 @@ jobs:
         echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV"
     - name: If no github.head_ref, use last chunk of github.ref as Slug
       if: github.head_ref == ''
-      run: | 
+      run: |
         IFS="\/";
-        read -a strarr <<< "${{ github.ref }}"; 
+        read -a strarr <<< "${{ github.ref }}";
         last_ref="${strarr[${#strarr[*]}-1]}";
         echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV"
     - name: Document which reference

.github/workflows/docs.yml

@@ -22,9 +22,9 @@ jobs:
         echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV"
     - name: If no github.head_ref, use last chunk of github.ref as Slug
       if: github.head_ref == ''
-      run: | 
+      run: |
         IFS="\/";
-        read -a strarr <<< "${{ github.ref }}"; 
+        read -a strarr <<< "${{ github.ref }}";
         last_ref="${strarr[${#strarr[*]}-1]}";
         echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV"
     - name: Document which reference

.github/workflows/validate_samples.yml

@@ -0,0 +1,72 @@
+name: Validate-Samples
+
+on:
+  push:
+    paths:
+      - 'samples/*/TIDES/*'
+      - 'spec/*'
+  pull_request:
+    paths:
+      - 'samples/*/TIDES/*'
+      - 'spec/*'
+  workflow_dispatch:
+  create:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+      - name: Install Frictionless
+        run: pip install frictionless[excel,json]>=5.0.0
+      - name: Find Sample Folders
+        id: sample_folders
+        run: |
+          pwd
+          samples=$(find ./samples -type d -name 'TIDES')
+          echo "{name}={value}" >> $GITHUB_OUTPUT
+      - name: Validate Sample Data
+        id: validation
+        run: |
+          samples=(${{ steps.sample_folders.outputs.samples }})
+          results=""
+          for sample in "${samples[@]}"; do
+            result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json)
+            results+="$result\n"
+          done
+          echo "{name}=${results}" >> $GITHUB_ENV
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const samples = '${{ env.samples }}'.trim().split('\n');
+            const results = '${{ env.results }}'.trim().split('\n');
+            let comment = `**Data Validation Report**\n\n`;
+            comment += '| Sample | Status |\n';
+            comment += '| ------ | ------ |\n';
+            for (let i = 0; i < samples.length; i++) {
+              const sample = samples[i];
+              const result = JSON.parse(results[i]);
+              let status = '';
+              if (result.valid) {
+                status = ':heavy_check_mark:';
+              } else if (result.stats.errorCount > 0) {
+                status = ':x:';
+              } else {
+                status = ':warning:';
+              }
+              comment += `| ${sample} | ${status} |\n`;
+            }
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: comment
+            });

.pre-commit-config.yaml

@@ -1,24 +1,33 @@
+default_stages: [push]
 repos:
   - repo: https://github.com/psf/black
     rev: 22.8.0
     hooks:
       - id: black
+        stages: [commit]
   - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
     - id: flake8
+      stages: [commit]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
+        stages: [commit]
       - id: end-of-file-fixer
+        stages: [commit]
       - id: check-yaml
         args: ["--unsafe"]
+        stages: [commit]
       - id: check-added-large-files
       - id: check-json
+        stages: [commit]
       - id: requirements-txt-fixer
+        stages: [commit]
       - id: mixed-line-ending
+        stages: [commit]
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.32.2
     hooks:

CONTRIBUTING.md

@@ -24,55 +24,19 @@ By making any contribution to the projects, contributors self-certify to the [Co
 1. Make sure you have a [GitHub](https://github.com/) account.  
 2. Make sure you have [git](https://git-scm.com/downloads), a terminal (e.g. Mac Terminal, CygWin, etc.), and a text editor installed on your local machine.  Optionally, you will likely find it easier to use [GitHub Desktop](https://desktop.github.com/), an IDE instead of a simple text editor like [VSCode](https://code.visualstudio.com/), [Eclipse](https://www.eclipse.org/), [Sublime Text](https://www.sublimetext.com/), etc.  
 3. [Fork the repository](https://github.com/TIDES-transit/TIDES/fork) into your own GitHub account and [clone it locally](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository).  
-4. Install development requirements packages `pip install -r requirements.txt`
+4. Install development requirements packages `pip install -r requirements.txt` or in a virtual environment.
+
+!!! tip "Using a virtual environment"
+
+    It is often helpful to install requirements for vairous projects inside a virtual environment rather than in your main python installation. Some virtual environments to consider include: [`conda`](https://docs.conda.io/),[`pipenv`](https://pipenv.pypa.io/en/latest/index.html),[`poetry`](https://python-poetry.org/), and [`rye`](https://github.com/mitsuhiko/rye).
 
 ### Contribution Workflow
 
 1. [Create a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) to work on a new issue (or checkout an existing one where the issue is being worked on).  
 2. Make your changes.
-3. Run `pre-commit run --all` to check and fix formatting.
-4. Validate schemas if you have changed them using either [act](https://github.com/nektos/act) or  [frictionless framework](https://framework.frictionlessdata.io/).
-5. [Commit](#commits) your work in `git`
-6. `push` your changes to Github and submit a [`pull request`](#pull-requests)
-
-!!! bug
-
-    Right now some github actions are broken due to some dependencies issues, so please validate changes to schemas using the [frictionless framework](https://framework.frictionlessdata.io/)
-
-??? tip "Manually Validating TIDES Schema Files with Frictionless"
-
-    === "CLI"
-
-        ```sh
-        frictionless validate spec/devices.schema.json
-        frictionless validate spec/fare_transactions.schema.json
-        frictionless validate spec/operators.schema.json
-        frictionless validate spec/passenger_events.schema.json
-        frictionless validate spec/station_activities.schema.json
-        frictionless validate spec/stop_visits.schema.json
-        frictionless validate spec/train_cars.schema.json
-        frictionless validate spec/trips_performed.schema.json
-        frictionless validate spec/vehicle_locations.schema.json
-        frictionless validate spec/vehicle_train_cars.schema.json
-        frictionless validate spec/vehicles.schema.json
-        ```
-
-    === "python"
-
-        ```python
-        import glob
-        from pprint import pprint
-        from frictionless import validate
-
-        schema_paths = glob.glob("spec/**.schema.json")
-        for p in schema_paths:
-            report = validate(p)
-            if not report['valid']:
-                print(f"!!! Invalid:{p}")
-                pprint(report)
-            else: 
-                print(f"Valid:{p}")
-        ```
+3. Run `/tests/test_all` script to check and fix formatting, validate schemas, and build documentation locally to preview
+4. [Commit](#commits) your work in `git`
+5. `push` your changes to Github and submit a [`pull request`](#pull-requests)
 
 ### Issues
 

main.py

@@ -57,9 +57,8 @@ def include_file(
             4: re.compile(r"(#{4}\s)(.*)"),
             5: re.compile(r"(#{5}\s)(.*)"),
         }
-        print(f"???before downshifting! {full_filename}")
+
         if md_heading_re[1].search(content) and downshift_h1:
-            print("!!!downshifting!")
             content = re.sub(md_heading_re[5], r"#\1\2", content)
             content = re.sub(md_heading_re[4], r"#\1\2", content)
             content = re.sub(md_heading_re[3], r"#\1\2", content)

mkdocs.yml

@@ -58,7 +58,7 @@ markdown_extensions:
   - pymdownx.tasklist:
       custom_checkbox: true
   - pymdownx.tabbed:
-      alternate_style: true 
+      alternate_style: true
   - pymdownx.superfences:
       custom_fences:
         - name: mermaid

requirements.txt

@@ -1,3 +1,5 @@
 -r docs/requirements.txt
-pre-commit
 frictionless
+jsonschema
+jsonschema-cli
+pre-commit

tests/test_all

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Runs pre-commit, then all the scripts in this dir, and then builds documentaiton locally
+
+# Get the list of script files in the directory
+script_files=("$(dirname "$(realpath "$0")")"/*)
+num_files=$((${#script_files[@]} + 1))
+test_num=1
+error_scripts=()
+
+echo "Found $num_files tests"
+
+echo "----- Running test 1 of $num_files: pre-commit"
+pre-commit run --all-files
+
+# Iterate over script files in the directory
+for script_file in "${script_files[@]}"; do
+
+  # Exclude the current script and non-executable files from execution
+  if [ "$(basename "$script_file")" != "test_all" ]; then
+    ((test_num++))
+    if [ -x "$script_file" ]; then
+      # Update progress
+      echo "----- Running test $test_num of $num_files: $script_file"
+
+      # Execute the script and capture errors if there are any
+      if output=$("$script_file" 2>&1); then
+        echo ":-) $script_file ran successfully."
+      else
+        echo "!!! $script_file encountered an error."
+        error_scripts+=("$script_file")
+        echo "$output" >&2
+      fi
+    else
+      # Notify non-executable files
+      echo "!!! File is not executable so was not run: $script_file."
+      echo
+    fi
+  fi
+done
+echo "----- Running test $num_files/$num_files: Building documentation locally"
+mkdocs build --quiet
+
+if [ ${#error_scripts[@]}  -gt 0 ]; then
+  echo "The following scripts executed with errors:"
+  for script in "${error_scripts[@]}"; do
+    echo "$script"
+  done
+fi

tests/validate_profile

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+echo "Validating tides-data-package.json profile"
+
+# Set the temporary directory path
+temp_dir=$(mktemp -d)
+
+# Download the schema file to the temporary directory
+schema_url="https://specs.frictionlessdata.io/schemas/data-package.json"
+schema_file="$temp_dir/data-package.json"
+curl -o "$schema_file" "$schema_url"
+
+# Validate datapackage against the downloaded schema
+datapackage_file="spec/tides-data.json"
+jsonschema-cli validate "$schema_file" "$datapackage_file"
+
+# Clean up the temporary directory
+rm -rf "$temp_dir"

tests/validate_samples

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+echo "Validating sample data"
+shopt -s nullglob
+for file in samples/*/TIDES/datapackage.json; do
+    echo "Validating: $file"
+    frictionless validate --schema-sync $file
+done
+shopt -u nullglob

tests/validate_table_schemas

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "Validating table schemas"
+
+echo "Processing file: $file"
+for file in spec/*schema.json; do
+  echo "Validating: $file"
+  frictionless validate $file
+done