diff --git a/.github/workflows/clean-docs.yml b/.github/workflows/clean-docs.yml index e975519b..9b1b2c05 100644 --- a/.github/workflows/clean-docs.yml +++ b/.github/workflows/clean-docs.yml @@ -14,9 +14,9 @@ jobs: echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV" - name: If no github.head_ref, use last chunk of github.ref as Slug if: github.head_ref == '' - run: | + run: | IFS="\/"; - read -a strarr <<< "${{ github.ref }}"; + read -a strarr <<< "${{ github.ref }}"; last_ref="${strarr[${#strarr[*]}-1]}"; echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV" - name: Document which reference diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 55ebb430..166775d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,9 +22,9 @@ jobs: echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV" - name: If no github.head_ref, use last chunk of github.ref as Slug if: github.head_ref == '' - run: | + run: | IFS="\/"; - read -a strarr <<< "${{ github.ref }}"; + read -a strarr <<< "${{ github.ref }}"; last_ref="${strarr[${#strarr[*]}-1]}"; echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV" - name: Document which reference diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml new file mode 100644 index 00000000..6e05dc2e --- /dev/null +++ b/.github/workflows/validate_samples.yml @@ -0,0 +1,72 @@ +name: Validate-Samples + +on: + push: + paths: + - 'samples/*/TIDES/*' + - 'spec/*' + pull_request: + paths: + - 'samples/*/TIDES/*' + - 'spec/*' + workflow_dispatch: + create: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install Frictionless + run: pip install frictionless[excel,json]>=5.0.0 + - name: Find Sample Folders + id: sample_folders + run: | + pwd + samples=$(find ./samples -type d -name 'TIDES') + echo "{name}={value}" >> $GITHUB_OUTPUT + - name: Validate Sample Data + id: validation + run: | + samples=(${{ steps.sample_folders.outputs.samples }}) + results="" + for sample in "${samples[@]}"; do + result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) + results+="$result\n" + done + echo "{name}=${results}" >> $GITHUB_ENV + - name: Comment on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const samples = '${{ env.samples }}'.trim().split('\n'); + const results = '${{ env.results }}'.trim().split('\n'); + let comment = `**Data Validation Report**\n\n`; + comment += '| Sample | Status |\n'; + comment += '| ------ | ------ |\n'; + for (let i = 0; i < samples.length; i++) { + const sample = samples[i]; + const result = JSON.parse(results[i]); + let status = ''; + if (result.valid) { + status = ':heavy_check_mark:'; + } else if (result.stats.errorCount > 0) { + status = ':x:'; + } else { + status = ':warning:'; + } + comment += `| ${sample} | ${status} |\n`; + } + github.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6a2c278..4ac8cd89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,24 +1,33 @@ +default_stages: [push] repos: - repo: https://github.com/psf/black rev: 22.8.0 hooks: - id: black + stages: [commit] - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: - id: flake8 + stages: [commit] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + stages: [commit] - id: end-of-file-fixer + stages: [commit] - id: check-yaml args: ["--unsafe"] + stages: [commit] - id: check-added-large-files - id: check-json + stages: [commit] - id: requirements-txt-fixer + stages: [commit] - id: mixed-line-ending + stages: [commit] - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.32.2 hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fddf83c5..38c82a11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,55 +24,19 @@ By making any contribution to the projects, contributors self-certify to the [Co 1. Make sure you have a [GitHub](https://github.com/) account. 2. Make sure you have [git](https://git-scm.com/downloads), a terminal (e.g. Mac Terminal, CygWin, etc.), and a text editor installed on your local machine. Optionally, you will likely find it easier to use [GitHub Desktop](https://desktop.github.com/), an IDE instead of a simple text editor like [VSCode](https://code.visualstudio.com/), [Eclipse](https://www.eclipse.org/), [Sublime Text](https://www.sublimetext.com/), etc. 3. [Fork the repository](https://github.com/TIDES-transit/TIDES/fork) into your own GitHub account and [clone it locally](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository). -4. Install development requirements packages `pip install -r requirements.txt` +4. Install development requirements packages `pip install -r requirements.txt` or in a virtual environment. + +!!! tip "Using a virtual environment" + + It is often helpful to install requirements for vairous projects inside a virtual environment rather than in your main python installation. Some virtual environments to consider include: [`conda`](https://docs.conda.io/),[`pipenv`](https://pipenv.pypa.io/en/latest/index.html),[`poetry`](https://python-poetry.org/), and [`rye`](https://github.com/mitsuhiko/rye). ### Contribution Workflow 1. [Create a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) to work on a new issue (or checkout an existing one where the issue is being worked on). 2. Make your changes. -3. Run `pre-commit run --all` to check and fix formatting. -4. Validate schemas if you have changed them using either [act](https://github.com/nektos/act) or [frictionless framework](https://framework.frictionlessdata.io/). -5. [Commit](#commits) your work in `git` -6. `push` your changes to Github and submit a [`pull request`](#pull-requests) - -!!! bug - - Right now some github actions are broken due to some dependencies issues, so please validate changes to schemas using the [frictionless framework](https://framework.frictionlessdata.io/) - -??? tip "Manually Validating TIDES Schema Files with Frictionless" - - === "CLI" - - ```sh - frictionless validate spec/devices.schema.json - frictionless validate spec/fare_transactions.schema.json - frictionless validate spec/operators.schema.json - frictionless validate spec/passenger_events.schema.json - frictionless validate spec/station_activities.schema.json - frictionless validate spec/stop_visits.schema.json - frictionless validate spec/train_cars.schema.json - frictionless validate spec/trips_performed.schema.json - frictionless validate spec/vehicle_locations.schema.json - frictionless validate spec/vehicle_train_cars.schema.json - frictionless validate spec/vehicles.schema.json - ``` - - === "python" - - ```python - import glob - from pprint import pprint - from frictionless import validate - - schema_paths = glob.glob("spec/**.schema.json") - for p in schema_paths: - report = validate(p) - if not report['valid']: - print(f"!!! Invalid:{p}") - pprint(report) - else: - print(f"Valid:{p}") - ``` +3. Run `/tests/test_all` script to check and fix formatting, validate schemas, and build documentation locally to preview +4. [Commit](#commits) your work in `git` +5. `push` your changes to Github and submit a [`pull request`](#pull-requests) ### Issues diff --git a/main.py b/main.py index 820c0711..fada5b1f 100644 --- a/main.py +++ b/main.py @@ -57,9 +57,8 @@ def include_file( 4: re.compile(r"(#{4}\s)(.*)"), 5: re.compile(r"(#{5}\s)(.*)"), } - print(f"???before downshifting! {full_filename}") + if md_heading_re[1].search(content) and downshift_h1: - print("!!!downshifting!") content = re.sub(md_heading_re[5], r"#\1\2", content) content = re.sub(md_heading_re[4], r"#\1\2", content) content = re.sub(md_heading_re[3], r"#\1\2", content) diff --git a/mkdocs.yml b/mkdocs.yml index bec3855a..aae3d55c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,7 +58,7 @@ markdown_extensions: - pymdownx.tasklist: custom_checkbox: true - pymdownx.tabbed: - alternate_style: true + alternate_style: true - pymdownx.superfences: custom_fences: - name: mermaid diff --git a/requirements.txt b/requirements.txt index 7043a64b..39184e02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -r docs/requirements.txt -pre-commit frictionless +jsonschema +jsonschema-cli +pre-commit diff --git a/tests/test_all b/tests/test_all new file mode 100755 index 00000000..cfd886da --- /dev/null +++ b/tests/test_all @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Runs pre-commit, then all the scripts in this dir, and then builds documentaiton locally + +# Get the list of script files in the directory +script_files=("$(dirname "$(realpath "$0")")"/*) +num_files=$((${#script_files[@]} + 1)) +test_num=1 +error_scripts=() + +echo "Found $num_files tests" + +echo "----- Running test 1 of $num_files: pre-commit" +pre-commit run --all-files + +# Iterate over script files in the directory +for script_file in "${script_files[@]}"; do + + # Exclude the current script and non-executable files from execution + if [ "$(basename "$script_file")" != "test_all" ]; then + ((test_num++)) + if [ -x "$script_file" ]; then + # Update progress + echo "----- Running test $test_num of $num_files: $script_file" + + # Execute the script and capture errors if there are any + if output=$("$script_file" 2>&1); then + echo ":-) $script_file ran successfully." + else + echo "!!! $script_file encountered an error." + error_scripts+=("$script_file") + echo "$output" >&2 + fi + else + # Notify non-executable files + echo "!!! File is not executable so was not run: $script_file." + echo + fi + fi +done +echo "----- Running test $num_files/$num_files: Building documentation locally" +mkdocs build --quiet + +if [ ${#error_scripts[@]} -gt 0 ]; then + echo "The following scripts executed with errors:" + for script in "${error_scripts[@]}"; do + echo "$script" + done +fi diff --git a/tests/validate_profile b/tests/validate_profile new file mode 100755 index 00000000..81f4ddd1 --- /dev/null +++ b/tests/validate_profile @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +echo "Validating tides-data-package.json profile" + +# Set the temporary directory path +temp_dir=$(mktemp -d) + +# Download the schema file to the temporary directory +schema_url="https://specs.frictionlessdata.io/schemas/data-package.json" +schema_file="$temp_dir/data-package.json" +curl -o "$schema_file" "$schema_url" + +# Validate datapackage against the downloaded schema +datapackage_file="spec/tides-data.json" +jsonschema-cli validate "$schema_file" "$datapackage_file" + +# Clean up the temporary directory +rm -rf "$temp_dir" diff --git a/tests/validate_samples b/tests/validate_samples new file mode 100755 index 00000000..cac8c226 --- /dev/null +++ b/tests/validate_samples @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +echo "Validating sample data" +shopt -s nullglob +for file in samples/*/TIDES/datapackage.json; do + echo "Validating: $file" + frictionless validate --schema-sync $file +done +shopt -u nullglob diff --git a/tests/validate_table_schemas b/tests/validate_table_schemas new file mode 100755 index 00000000..7af96f9c --- /dev/null +++ b/tests/validate_table_schemas @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +echo "Validating table schemas" + +echo "Processing file: $file" +for file in spec/*schema.json; do + echo "Validating: $file" + frictionless validate $file +done