diff --git a/.github/workflows/validate_package_schema.yml b/.github/workflows/validate_package_schema.yml index c4d44051..fcab3cb3 100644 --- a/.github/workflows/validate_package_schema.yml +++ b/.github/workflows/validate_package_schema.yml @@ -17,7 +17,7 @@ jobs: uses: TIDES-transit/json-schema-validator@master with: token: ${{ secrets.GITHUB_TOKEN }} - json_schema: spec/tides-data-package.json + json_schema: spec/tides-datapackage-profile.json json_path_pattern: ^spec/tides\.spec\.json$ send_comment: false clear_comments: false diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 6e05dc2e..0e77c164 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -29,31 +29,47 @@ jobs: run: | pwd samples=$(find ./samples -type d -name 'TIDES') - echo "{name}={value}" >> $GITHUB_OUTPUT + echo $samples + echo "samples=${samples}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - results="" + results="{\"values\": [" + result="" for sample in "${samples[@]}"; do - result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) - results+="$result\n" + results+="$result" + result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true + result="${result//\'s/}" + result="${result//\\\"/\\\\\"}" + result="$result," done - echo "{name}=${results}" >> $GITHUB_ENV + + result="${result%?}" + results+="$result" + results+="]}" + + echo "RESULTS<> $GITHUB_OUTPUT + echo ${results} >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const samples = '${{ env.samples }}'.trim().split('\n'); - const results = '${{ env.results }}'.trim().split('\n'); + console.log("Begin sample construction") + const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); + const results = JSON.parse('${{ steps.validation.outputs.results }}'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; + console.log("Constructed header") for (let i = 0; i < samples.length; i++) { + console.log(samples[i]) + console.log(results[i]) const sample = samples[i]; - const result = JSON.parse(results[i]); + const result = results.values[i]; let status = ''; if (result.valid) { status = ':heavy_check_mark:'; diff --git a/.github/workflows/validate_table_schemas.yml b/.github/workflows/validate_table_schemas.yml index f64da12b..d65e1942 100644 --- a/.github/workflows/validate_table_schemas.yml +++ b/.github/workflows/validate_table_schemas.yml @@ -21,4 +21,4 @@ jobs: json_schema: spec/table-schema.json json_path_pattern: ^spec/.*\.schema\.json$ send_comment: false - clear_comments: false \ No newline at end of file + clear_comments: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 785c6026..7047c7d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,9 +34,10 @@ By making any contribution to the projects, contributors self-certify to the [Co 1. [Create a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) to work on a new issue (or checkout an existing one where the issue is being worked on). 2. Make your changes. -3. Run `/tests/test_all` script to check and fix formatting, validate schemas, and build documentation locally to preview -4. [Commit](#commits) your work in `git` -5. `push` your changes to Github and submit a [`pull request`](#pull-requests) +3. Run `tests/test_local_spec` script to check and fix formatting, validate profile and schemas with frictionless and with each other, and confirm that documentation can be built locally. +4. Run `tests/test_samples_to_local` script to check if samples conform to any changes to the spec. +5. [Commit](#commits) your work in `git` +6. `push` your changes to Github and submit a [`pull request`](#pull-requests) ### Issues @@ -105,7 +106,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de | **Name** | **What it does** | | -------- | ----------------- | | GitHub Actions | Runs following workflow on each push to the TIDES github repository: /.github/workflows/docs.yml | - | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc)
For new branches with documentation, adds an entry in `versions.json` | + | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc)
For new branches with documentation, adds an entry in `versions.json` | | `mkdocs` | Package which generates documentation from markdown and code | ??? info "Overview of Documentation Building Process" @@ -116,7 +117,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de subgraph mkdocs["mkdocs: run on execution of mike"] md_mike["mike"] -->|runs for current branch| md_mkdocs["mkdocs"] md_mkdocs.yml["mkdocs.yml"] -->|specifieds parameters| md_mkdocs["mkdocs"] - md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"] + md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"] main.py[/"main.py"/] -->|defines macros in code available for| md_mkdocs_macros["mkdocs-macros"] end diff --git a/README.md b/README.md index 62de1b4b..5f0dec1d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,35 @@ Human-friendlier documentation is auto-generated and available at: - [Architecture](https://tides-transit.github.io/TIDES/main/architecture) - [Table Schemas](https://tides-transit.github.io/TIDES/main/tables) +### Data Package + +Directories with TIDES data must contain metadata in a [`datapackage.json`](https://tides-transit.github.io/TIDES/main/datapackage) file in a format compliant with the [`tides-datapackage-profile`](https://tides-transit.github.io/TIDES/main/datapackage) of a [`frictionless data package`](https://specs.frictionlessdata.io/data-package/). + +[`/samples/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/datapackage.json) has a template datapackage which can be used. + +## Sample Data + +[Sample data](https://tides-transit.github.io/TIDES/main/samples) can be found in the `/samples` directory, with one directory for each sample. + +### Template + +Templates of `datapackage.json` and each TIDES file type are located in the `/samples/template` directory. + +## Validating TIDES data + +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: + +```bash +pip install frictionless +frictionless validate --schema-sync path/to/your/datapackage.json +``` + +Several other validation scripts and tools with more flexibility such as validating to the canonical, named version or a local spec can be found in the `/bin` directory, with usage available with the `--help` flag. + +```bash +bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path] +``` + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/bin/replace-spec-in-datapackage b/bin/replace-spec-in-datapackage new file mode 100755 index 00000000..889800ed --- /dev/null +++ b/bin/replace-spec-in-datapackage @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +description="Create a temporary data package if a spec_path_prefix is provided." + +usage=" +Usage: replace-spec-in-datapackage [spec_path_prefix] [output_file] + +Arguments: + The path to the dataset directory containing the 'datapackage.json' file. + The path or URL to the spec to be referenced in the updated data package. + [output_file] (Optional) The path to save the temporary data package. If not provided, the temporary data package will be saved as 'datapackage.tmp.json' in the dataset directory. + +" + +example_usage=" +Example Usage: +bin/replace-spec-in-datapackage samples/template/TIDES spec samples/template/TIDES/datapackage.tmp.json +" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# SET DEFAULTS # +################################################################################ +DEFAULT_TMP_DATAPACKAGE="datapackage.tmp.json" +PROFILE_FILE="tides-datapackage-profile.json" + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Check Requirements # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +check_jq + +################################################################################ +# Process the input options. # +################################################################################ +dataset_path=$1 +spec_path_prefix=$2 +output_file=${3:-"$dataset_path/$DEFAULT_TMP_DATAPACKAGE"} + +echo "Parameters: + dataset_path: $dataset_path + spec_path_prefix: $spec_path_prefix + output_file: $output_file +" + +# Check if required arguments are missing +if [ -z "$dataset_path" ]; then + echo "Error: Missing dataset_path argument." >&2 + display_help + exit 1 +fi + +# Check if required arguments are missing +if [ -z "spec_path_prefix" ]; then + echo "Error: Missing spec_path_prefix argument." >&2 + display_help + exit 1 +fi + +datapackage_file="$dataset_path/datapackage.json" + +check_valid_path "$datapackage_file" + +profile_file="$spec_path_prefix/$PROFILE_FILE" +check_valid_path "$profile_file" + +################################################################################ +# Create updated datapackage # +################################################################################ +cp "$datapackage_file" "$output_file" +jq --arg spec_path_prefix "$spec_path_prefix" --arg profile_file "$profile_file" ' + .resources |= map(.schema |= ($spec_path_prefix + "/\(. | split("/") | last)")) + | .profile = ($profile_file) +' "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file" + +echo "$output_file" \ No newline at end of file diff --git a/bin/utils b/bin/utils new file mode 100755 index 00000000..75f5a05a --- /dev/null +++ b/bin/utils @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Check if jsonschema-cli is installed +check_jsonschema-cli() { + if ! command -v jsonschema-cli >/dev/null 2>&1; then + echo >&2 "\033[31m!!! jsonschema-cli is required but not found.\033[0m + + You can install it using 'pip install jsonschema-cli'. Aborting." + exit 1 + fi +} + +# Check if frictionless is installed +check_frictionless() { + if ! command -v frictionless >/dev/null 2>&1; then + echo >&2 "\033[31m!!! frictionless is required but not found.\033[0m + + You can install it using 'pip install frictionless'. Aborting." + exit 1 + fi +} + +# Function to check jq and provide installation instructions if not found +check_jq() { + if ! command -v jq >/dev/null 2>&1; then + echo >&2 "\033[31m!!! jq is required but not found.\033[0m" + + # Determine the operating system + os_type=$(uname -s) + + case "$os_type" in + Linux*) + echo >&2 "Please install jq using your package manager." + echo >&2 "For example, on Debian/Ubuntu-based systems, you can use:" + echo >&2 "sudo apt-get install jq" + ;; + Darwin*) + echo >&2 "Please install jq using Homebrew." + echo >&2 "If you don't have Homebrew installed, you can install it from https://brew.sh/." + echo >&2 "Once Homebrew is installed, run the following command:" + echo >&2 "brew install jq" + ;; + CYGWIN*|MINGW32*|MSYS*|MINGW*) + echo >&2 "Please download the jq binary for Windows from https://stedolan.github.io/jq/download/." + echo >&2 "Extract the downloaded ZIP file and add the 'jq.exe' binary to your PATH..." + echo >&2 "Or run the following command: " + echo >&2 "curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe" + ;; + *) + echo >&2 "Unknown operating system. Please install jq from https://stedolan.github.io/jq/download/." + ;; + esac + + exit 1 + fi +} + +# Function to check if this is a valid spec path +check_valid_path() { + local path=$1 + + if [ -n "$path" ]; then + if [ ! -e "$path" ] && [[ ! "$path" =~ ^http ]]; then + echo "\033[31mError: $path is an invalid path. It must be a valid file path, directory path, or a URL starting with 'http'.\033[0m" >&2 + display_help + exit 1 + elif [[ "$path" =~ ^http ]]; then + # Check if the URL is valid + if ! curl --output /dev/null --silent --head --fail "$path"; then + echo "\033[31mError: Invalid URL. The $path URL is not reachable or does not exist.\033[0m" >&2 + exit 1 + fi + fi + fi +} diff --git a/bin/validate-datapackage b/bin/validate-datapackage new file mode 100755 index 00000000..a6fbbcff --- /dev/null +++ b/bin/validate-datapackage @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +description="Bash script to validate a Frictionless Data Package using the Frictionless CLI." + +usage=" +Usage: validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path] + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile. Should not be used with -l option. Example: `-r main` + -l local_spec_path: Optional. Specify the path of the local schema directory. + Default is '../spec'. Is only used if remote_spec_ref = local. + -d dataset_path: Optional. Specify the path of the TIDES datapackage.json. + Default is the current directory. +" + +example_usage="Example Usage: bin/validate-datapackage -l spec -d samples/template/TIDES" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Source utility functions # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +script_dir=$(dirname "$0") + +################################################################################ +# Check Requirements # +################################################################################ +check_frictionless + +################################################################################ +# Process the parameters + input options. # +################################################################################ +remote_spec_ref="" +local_spec_path="" +dataset_path="." + +# Parse command-line arguments +while getopts ":r:l:d:" opt; do + case $opt in + r) + remote_spec_ref=$OPTARG + ;; + l) + local_spec_path=$OPTARG + ;; + d) + dataset_path=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +echo "Running validate-datapackage with configs:" +echo " remote_spec_ref: $remote_spec_ref" +echo " local_spec_path: $local_spec_path" +echo " dataset_path: $dataset_path" + +if [ "$local_spec_path" != "" ] && [ "$remote_spec_ref" != "" ]; then + echo "Cannot specify both local profile location (-l) and remote spec reference (-r)" >&2 + exit 1 +fi + +# Convert local_spec_path to an absolute path +if [ -n "$local_spec_path" ]; then + local_spec_path=$(realpath "$local_spec_path") +fi + +################################################################################ +# Specify datapackage and update if necessary # +################################################################################ + +datapackage_file="$dataset_path/datapackage.json" +check_valid_path "$datapackage_file" + +[ -n "$local_spec_path" ] && spec_path_prefix="$local_spec_path" +[ -n "$remote_spec_ref" ] && spec_path_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$remote_spec_ref/spec" + +# Create a temporary data package if using a different spec than specified in datapackage +if [ "$spec_path_prefix" != "" ]; then + tmp_datapackage=$("$script_dir/replace-spec-in-datapackage" "$dataset_path" "$spec_path_prefix" 2>&1 | tee /dev/tty | tail -1) + echo "...using Temporary datapackage at: $tmp_datapackage" + check_valid_path "$tmp_datapackage" + datapackage_file="$tmp_datapackage" +fi + +################################################################################ +# Validate datapackage file against TIDES profile # +################################################################################ + +echo "...validate the data package JSON against the TIDES profile" + +if ! "$script_dir"/validate-datapackage-to-profile -d "$datapackage_file"; then + echo -e "\033[31m!!! ERROR: Data package validation against the TIDES profile failed.\033[0m" +fi + +################################################################################ +# Validate datapackage contents # +################################################################################ +echo "...validate the Frictionless Data Package using the Frictionless CLI" +frictionless validate --schema-sync "$datapackage_file" --trusted + +# Remove the temporary data package file, if applicable +if [ "$tmp_datapackage" != "" ]; then + rm "$tmp_datapackage" +fi diff --git a/bin/validate-datapackage-to-profile b/bin/validate-datapackage-to-profile new file mode 100755 index 00000000..24d92feb --- /dev/null +++ b/bin/validate-datapackage-to-profile @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +description="Script to validate a local JSON file against a profile for tides-data-package specified in +profile field or optionally against a remote or local profile." + +usage=" +Usage: validate-datapackage-to-profile [-r remote_spec_ref | -l local_spec_path] [-f datapackage_file] + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile where the profile is in the sub-path /spec/tides-data-package.json. + Should not be used with -l option. Example: -r main + -l local_spec_path: Optional. Specify the location of the local tides-data-package-json to use. + Should not be used with -r option. Example: -l spec + -d dataset_path: Optional. Specify the path of the datapackage.json file. + Default is datapackage.json. Example: -d samples/template/TIDES/datapackage.json +" + +example_usage="Example Usage: bin/validate-datapackage-to-profile -d samples/template/TIDES/datapackage.tmp.json" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Source utility functions # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +script_dir=$(dirname "$0") + +################################################################################ +# Check Requirements # +################################################################################ +check_jsonschema-cli +check_jq + +################################################################################ +# Process the input options. # +################################################################################ +# Set default values +remote_spec_ref="" +local_spec_path="" +dataset_path="datapackage.json" + +# Parse command-line arguments +while getopts ":r:l:d:" opt; do + case $opt in + r) + remote_spec_ref=$OPTARG + ;; + l) + local_spec_path=$OPTARG + ;; + d) + dataset_path=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +echo "validate-datapackage-to-profile configs:" +echo " remote_spec_ref: $remote_profile_ref" +echo " local_spec_path: $local_profile_path" +echo " dataset_path: $dataset_path" + +if [ "$local_spec_path" != "" ] && [ "$remote_spec_path" != "" ]; then + echo "Cannot specify both local spec location (-l) and remote spec reference (-r)" >&2 + exit 1 +fi + +################################################################################ +# Specify datapackage and update if necessary # +################################################################################ + +# Set appropriate datapackage_file vs dataset_path +if [ -f "$dataset_path" ]; then + datapackage_file="$dataset_path" + dataset_path=$(dirname "$dataset_path") +else + datapackage_file="$dataset_path/datapackage.json" +fi +check_valid_path "$datapackage_file" + +################################################################################ +# Find and download if necessary the profile file # +################################################################################ + +[ -n "$local_spec_path" ] && spec_prefix="$local_spec_path" +[ -n "$remote_spec_ref" ] && spec_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$remote_spec_ref/spec" + +if [ "$spec_prefix" != "" ]; then + profile_file="$spec_prefix/tides-datapackage-profile.json" +else + profile_file=$(jq -r '.profile' "$datapackage_file") +fi +echo "...using profile: $profile_file" +check_valid_path "$profile_file" + +if [[ "$profile_file" =~ ^http ]]; then + temp_dir=$(mktemp -d) + profile_file="$temp_dir/$(basename $profile_file)" + echo "...downloading profile to: $profile_file" + curl -L "$profile_file" -o "$profile_file" +fi + +################################################################################ +# Validate Datapackage to Profile # +################################################################################ +echo "...validating $datapackage_file to $profile_file" +jsonschema-cli validate "$profile_file" "$datapackage_file" + +################################################################################ +# Cleanup # +################################################################################ +if [ -d "$temp_dir" ]; then + rm -r "$temp_dir" +fi diff --git a/contributors/index.html b/contributors/index.html new file mode 100644 index 00000000..1ba8159d --- /dev/null +++ b/contributors/index.html @@ -0,0 +1,15 @@ + + + + + + Redirecting... + + + + + + +Redirecting... + + diff --git a/docs/datapackage.md b/docs/datapackage.md new file mode 100644 index 00000000..c6cb04ec --- /dev/null +++ b/docs/datapackage.md @@ -0,0 +1,75 @@ +# Data Package + +TIDES data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json), which is an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/) schema. + +You may create your own `datapackage.json` based on the documentaiton or start with the provided [template](#template), but don't forget to [validate](#validation) it to make sure it is in the correct format! + +## Data Package Format + +{{ frictionless_data_package('spec/tides-datapackage-profile.json') }} + +## Tabular Data Resource + +Required and recommended fields for each `tabluar-data-resource` are as follows: + +{{ frictionless_data_package('spec/tides-datapackage-profile.json',sub_schema="resources") }} + +## Template + +The canonical `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/TIDES/datapackage.json). + +!!! warning + This version of `tides-data-package` template is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` template if you are viewing the `main` documentation version. + +{{ include_file('samples/template/TIDES/datapackage.json',code_type='json') }} + +## Validation + +There are lots of options for validating your `datapackage.json` file including: + +- [Command Line Interface (CLI) Script](#cli) +- [Various online websites](#point-and-drool) + +### CLI + +You can easily validate your data package file with the script provided in [`/bin/validate-data-package-to-profile`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/bin/validate-data-package-to-profile) + +??? tip "installation requirements" + + Make sure you have jsonschema-cli installed. You can install it specifically or with all of the other suggested tools using one of the commands below: + + === "Mac" + + ```sh + brew install jq # mac + pip install -r requirements.txt + ``` + + === "Windows" + + ```sh + curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe #windows + pip install -r requirements.txt + ``` + + === "Linux" + + ```sh + sudo apt-get install jq # linux + pip install -r requirements.txt + ``` + +```sh title="usage" +validate-data-package-json -f my-datapackage.json +``` + +{{ include_file('bin/validate-datapackage-to-profile',code_type='sh') }} + +### Point-and-Drool + +Because a `tides-datapackage-profile` is just a json-schema, you can use the myriad of different json-schema validator out there on the web. Use the [canonical `tides-datapackage-profile`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json) or copy and paste the version from below. + +!!! warning + This version of `tides-datapackage-profile` is dependent on the version of the documentation you are viewing and only represents the canonical `tides-datapackage-profile` if you are viewing the `main` documentation version. + +{{ include_file('spec/tides-datapackage-profile.json',code_type='json') }} diff --git a/docs/samples.md b/docs/samples.md new file mode 100644 index 00000000..7b4125ca --- /dev/null +++ b/docs/samples.md @@ -0,0 +1,9 @@ +# Sample Data + +Sample data can be found in the `/samples` directory, with one directory for each data sample. + +{{ include_file('samples/README.md')}} + +## Data List + +{{ list_samples('samples') }} diff --git a/main.py b/main.py index fada5b1f..38d9037b 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,15 @@ import glob import json +import logging import os import pathlib import re -from typing import Union +from typing import Union, Literal import pandas as pd +log = logging.getLogger("mkdocs") + FIND_REPLACE = { # original relative to /docs : redirect target "": "[Contributing Section](development/#CONTRIBUTING)", "(CODE_OF_CONDUCT.md)": "(development/#CODE_OF_CONDUCT)", @@ -26,9 +29,39 @@ def define_env(env): - macro: a decorator function, to declare a macro. """ + @env.macro + def list_samples(sample_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + Args: + sample_dir (str):directory to search in + Returns: + str: markdown-formatted list + """ + EXCLUDE = ["template"] + fields = ["Sample", "Agency", "Resources", "Vendors"] + + sample_dir = os.path.join(env.project_dir, sample_dir) + samples = [ + d + for d in os.listdir(sample_dir) + if os.path.isdir(os.path.join(sample_dir, d)) and d not in EXCLUDE + ] + + md_table = ( + "| *" + "** | **".join(fields) + "* |\n| " + " ----- |" * len(fields) + "\n" + ) + + for s in samples: + md_table += datapackage_to_row(os.path.join(sample_dir, s, "TIDES"), fields) + return md_table + @env.macro def include_file( - filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None + filename: str, + downshift_h1=True, + start_line: int = 0, + end_line: int = None, + code_type: str = None, ): """ Include a file, optionally indicating start_line and end_line. @@ -42,6 +75,8 @@ def include_file( start_line (Optional): if included, will start including the file from this line (indexed to 0) end_line (Optional): if included, will stop including at this line (indexed to 0) + code_type: if not None, will encapsulate the resulting file in + ```..file contents...``` """ full_filename = os.path.join(env.project_dir, filename) with open(full_filename, "r") as f: @@ -71,26 +106,118 @@ def include_file( _replace = _replace.replace(_filenamebase, "") content = content.replace(_find, _replace) + + if code_type is not None: + content = f"\n```{code_type} title='{filename}'\n{content}\n```" return content + @env.macro + def frictionless_data_package( + data_package_path: str = "**/data-package.json", + sub_schema: str = None, + include: Literal["required", "recommended", "all"] = "recommended", + ) -> str: + """Describes top-level fields of a data package as a markdown table. + + Note: Right now if it finds more than one, it will take the first one. + + Args: + data_package_path (str, optional): location or glob pattern of data package. If glob, + will use the first found match. + sub_schema: if provided, will look for that portion of the schema and use that as + the "top level" + include: specifies the fields to include. Must be one of: + - "required": will only document required fields + - "recommended": will document required and recommended fields + - "all": will document all fields + Defaults to "recommended". + + Returns: a markdown table string + """ + INCLUDE = ["name", "description", "type", "requirement"] + + dp_filename = glob.glob(data_package_path, recursive=True)[0] + + log.info( + f"Documenting spec_file: {dp_filename}.{sub_schema} including {include}" + ) + + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + if sub_schema is not None: + if sub_schema in dp["properties"]: + dp = dp["properties"][sub_schema] + elif sub_schema in dp["$defs"]: + dp = dp["$defs"][sub_schema] + else: + return "Cannot find sub-schema {sub_schema} in data package file {dp_filename}" + if dp["type"] == "array": + dp = dp["items"] + + _field_names = [] + + if include in ["required", "recommended"]: + _field_names += dp.get("required", []) + if include == "recommended": + _field_names += dp.get("recommended", []) + if include == "all": + _field_names = list(dp["properties"].keys()) + + if not _field_names: + return "No fields found to document with parameters." + + _fields = [] + for _f in _field_names: + log.debug(f"Documenting: {_f}") + _row_entry = {k: v for k, v in dp["properties"][_f].items() if k in INCLUDE} + if _f in dp.get("required", []): + _row_entry["requirement"] = "required" + elif _f in dp.get("recommended", []): + _row_entry["requirement"] = "recommended" + else: + _row_entry["requirement"] = " - " + + _row_entry["name"] = f"`{_f}`" + if dp["properties"][_f].get("enum"): + _row_entry["description"] += "
**Must** be one of:
  • `" + _row_entry["description"] += "`
  • `".join( + dp["properties"][_f]["enum"] + ) + _row_entry["description"] += "`
" + elif dp["properties"][_f].get("const"): + _row_entry[ + "description" + ] += f"
**Must** be: `{dp['properties'][_f]['const']}`" + elif dp["properties"][_f].get("examples"): + _ex = dp["properties"][_f]["examples"][0].replace("\n", "
") + _row_entry["description"] += f"
**Example**:
`{_ex}`" + + _fields.append(_row_entry) + + dp_df = pd.DataFrame(_fields, columns=INCLUDE) + dp_md = dp_df.to_markdown(index=False) + + return dp_md + @env.macro def frictionless_spec( spec_path: str = "**/*.spec.json", ) -> str: """Translate the frictionless .spec file to a markdown table. - Note: Right now if it finds more than one, it will take the first one. + Note: Right now if it finds more than one, it will take the first one. If glob, + will use the first found match. Args: - spec_path (str, optional): base path of repo. Defaults to two directories - up from this file. + spec_path (str, optional): location or glob pattern of spec Returns: a markdown table string """ spec_file = glob.glob(spec_path, recursive=True)[0] - print("Documenting spec_file: ", spec_file) + log.info(f"Documenting spec_file: {spec_file}") # Generate a table for overall file requirements spec_df = read_config(spec_file) @@ -117,7 +244,7 @@ def frictionless_schemas( # Create markdown with a table for each schema file schema_files = glob.glob(schema_path, recursive=True) - print("Documenting schemas from: {}".format(schema_files)) + log.info("Documenting schemas from: {schema_files}") file_schema_markdown = [ _document_frictionless_schema(_s) for _s in schema_files @@ -136,6 +263,49 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] +def datapackage_to_row(dir: str, fields: list) -> str: + """Reads datapackage and translates key metadata to a row in a markdown table. + + Right now, only works for fields: "Sample", "Agency", "Resources", "Vendors" + + Args: + dir (str): fully qualified directory for sample data + fields (list): list of fields to return + + Returns: + str: row in a markdown table with name, agency, resources, vendors + """ + dp_filename = os.path.join(dir, "datapackage.json") + if not os.path.isfile(dp_filename): + raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + _vendors = [ + s.get("vendor", None) for r in dp["resources"] for s in r.get("sources", []) + ] + _vendors = list(set(_vendors) - set([None])) + + _md_cells = { + "Sample": _to_sample_readme_link(dp["name"], dir), + "Agency": dp["agency"], + "Resources": ( + "
  • `" + + "`
  • `".join([r["name"] for r in dp["resources"]]) + + "`
" + ), + "Vendors": "
  • `" + "`
  • `".join(_vendors) + "`
", + } + + md_row = "| " + " | ".join([_md_cells[f] for f in fields]) + " |\n" + + return md_row + + +def _to_sample_readme_link(sample_name, folder_dir): + return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" + + def _document_frictionless_schema(schema_filename: str) -> dict: """Documents a single frictionless schema. @@ -149,7 +319,7 @@ def _document_frictionless_schema(schema_filename: str) -> dict: "schema_md": markdown table documenting fields """ - print(f"Documenting schema from file: {schema_filename}") + log.info(f"Documenting schema from file: {schema_filename}") spec_name = schema_filename.split("/")[-1].split(".")[0] spec_title = " ".join([x.capitalize() for x in spec_name.split("_")]) schema = read_schema(schema_filename) diff --git a/mkdocs.yml b/mkdocs.yml index aae3d55c..644883c3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,6 +6,7 @@ edit_uri: edit/main/docs theme: name: material features: + - content.code.copy - navigation.tabs - navigation.expand - toc.integrate @@ -36,6 +37,7 @@ plugins: securityLevel: 'loose' - search + watch: - CONTRIBUTING.md - CODE_OF_CONDUCT.md @@ -74,4 +76,6 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md + - Data Package: datapackage.md + - Sample Data: samples.md - Development: development.md diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..66dc9564 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,69 @@ +# Samples Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +```sh +unique-example-name + \TIDES # Required. Data to be validated against the TIDES specification + \datapackage.json # Required. [`tides-data-package`](#tides-data-package) metadata + \raw # Optional. Data which the agency uses to create TIDES data + \scripts # Optional. Scripts used to transform raw --> TIDES +``` + +## Adding Examples + +We encourage the addition of examples, but please follow the following guidelines: + +1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. All individual files should be well under 50 MB. +2. *Include Metadata* as specified in [tides-data-package](#data-package). +3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. + +## Data Package + +TIDES sample data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json) (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). + +See: + +- [Full documentation on the `tides-data-package`]({{ site_url }}/datapackage) +- [Example `tides-data-package` file](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template) + +## Data validation + +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: + +```bash +pip install frictionless +frictionless validate --schema-sync path/to/your/datapackage.json +``` + +Alternatively, we also provide a wrapper script to provide some additional flexibility and options. + +Usage: `bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path]` + +- `-r remote_spec_ref`: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile. Should not be used with -l option. Example: `-r main` +- `-l local_spec_path`: Optional. Specify the path of the local schema directory. + Default is 'spec'. Is only used if remote_spec_ref = local. +- `-d dataset_path`: Optional. Specify the path of the TIDES datapackage.json. + Default is the current directory. + +Key usage examples: + +- `bin/validate-datapackage -l spec -d samples//TIDES`: Validate my sample data to a version of the spec located in the `/spec` directory. +- `bin/validate-datapackage -r main -d samples//TIDES`: Validate my sample data to the canonical version of the TIDES spec. +- `bin/validate-datapackage -r v1.0 -d samples//TIDES`: Validate my sample data to v1.0 of the spec. +- `bin/validate-datapackage -r develop -d samples//TIDES`: Validate my sample data to the current `develop` branch of the TIDES spc. + +If you **only** want to validate your `datapackage.json` file and not the datapackage as a whole, you can run the script `bin/validate-datapackage-to-profile` instead with the same options. Note that this is also run as a part of `validate-datapackage`. + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json +``` + +### Continuous Data Validation + +Sample data in the `\TIDES` subdirectories of each sample is validated upon a push action to the main repository. diff --git a/samples/template/README.md b/samples/template/README.md new file mode 100644 index 00000000..1cdaf7f0 --- /dev/null +++ b/samples/template/README.md @@ -0,0 +1,16 @@ +# Example TIDES Data Package + +Template directory for example scaffolding and helper scripts. + +## Scripts for Generating Template Data + +`scripts\create_template.py` has some template code which can help with the following + +- `write_schema_examples()`: will generate blank csvs according to the TIDES schema +- `write_datapackage()` will generate a datapackage.json based on the TIDES schemas and a set of defaults specified in th script. + +To run both (note this replaces the existing files in the directory) + +```bash +python samples/template/scripts/create_template.py +``` diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json new file mode 100644 index 00000000..c667a37a --- /dev/null +++ b/samples/template/TIDES/datapackage.json @@ -0,0 +1,182 @@ +{ + "name": "template", + "title": "Template TIDES Data Package Example", + "agency": "Transit Agency Name", + "ntd_id": "1234-56", + "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json", + "licenses": [ + { + "name": "Apache-2.0" + } + ], + "contributors": [ + { + "title": "My Name", + "email": "me@myself.com", + "github": "myhandle" + } + ], + "maintainers": [ + { + "title": "Another Name", + "email": "another@myself.com", + "github": "myhandle" + } + ], + "resources": [ + { + "name": "devices", + "profile": "tabular-data-resource", + "path": "devices.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_locations", + "profile": "tabular-data-resource", + "path": "vehicle_locations.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "fare_transactions", + "profile": "tabular-data-resource", + "path": "fare_transactions.csv", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "train_cars", + "profile": "tabular-data-resource", + "path": "train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "operators", + "profile": "tabular-data-resource", + "path": "operators.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "stop_visits", + "profile": "tabular-data-resource", + "path": "stop_visits.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_train_cars", + "profile": "tabular-data-resource", + "path": "vehicle_train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicles", + "profile": "tabular-data-resource", + "path": "vehicles.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "trips_performed", + "profile": "tabular-data-resource", + "path": "trips_performed.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "station_activities", + "profile": "tabular-data-resource", + "path": "station_activities.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "passenger_events", + "profile": "tabular-data-resource", + "path": "passenger_events.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + } + ] +} diff --git a/samples/template/TIDES/devices.csv b/samples/template/TIDES/devices.csv new file mode 100644 index 00000000..f1096555 --- /dev/null +++ b/samples/template/TIDES/devices.csv @@ -0,0 +1 @@ +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,device_location diff --git a/samples/template/TIDES/fare_transactions.csv b/samples/template/TIDES/fare_transactions.csv new file mode 100644 index 00000000..ef3b7eea --- /dev/null +++ b/samples/template/TIDES/fare_transactions.csv @@ -0,0 +1 @@ +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance,service_date,event_timestamp diff --git a/samples/template/TIDES/operators.csv b/samples/template/TIDES/operators.csv new file mode 100644 index 00000000..713abf28 --- /dev/null +++ b/samples/template/TIDES/operators.csv @@ -0,0 +1 @@ +operator_id diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv new file mode 100644 index 00000000..d4ece193 --- /dev/null +++ b/samples/template/TIDES/passenger_events.csv @@ -0,0 +1 @@ +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp diff --git a/samples/template/TIDES/station_activities.csv b/samples/template/TIDES/station_activities.csv new file mode 100644 index 00000000..3928c0fb --- /dev/null +++ b/samples/template/TIDES/station_activities.csv @@ -0,0 +1 @@ +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits,service_date diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv new file mode 100644 index 00000000..ecc6c3b7 --- /dev/null +++ b/samples/template/TIDES/stop_visits.csv @@ -0,0 +1 @@ +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date diff --git a/samples/template/TIDES/train_cars.csv b/samples/template/TIDES/train_cars.csv new file mode 100644 index 00000000..3b0e604b --- /dev/null +++ b/samples/template/TIDES/train_cars.csv @@ -0,0 +1 @@ +train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type diff --git a/samples/template/TIDES/trips_performed.csv b/samples/template/TIDES/trips_performed.csv new file mode 100644 index 00000000..94ce7bcd --- /dev/null +++ b/samples/template/TIDES/trips_performed.csv @@ -0,0 +1 @@ +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship,service_date diff --git a/samples/template/TIDES/vehicle_locations.csv b/samples/template/TIDES/vehicle_locations.csv new file mode 100644 index 00000000..f7936518 --- /dev/null +++ b/samples/template/TIDES/vehicle_locations.csv @@ -0,0 +1 @@ +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship,event_timestamp diff --git a/samples/template/TIDES/vehicle_train_cars.csv b/samples/template/TIDES/vehicle_train_cars.csv new file mode 100644 index 00000000..9044359c --- /dev/null +++ b/samples/template/TIDES/vehicle_train_cars.csv @@ -0,0 +1 @@ +vehicle_id,train_car_id,order,operator_id diff --git a/samples/template/TIDES/vehicles.csv b/samples/template/TIDES/vehicles.csv new file mode 100644 index 00000000..67ef59e8 --- /dev/null +++ b/samples/template/TIDES/vehicles.csv @@ -0,0 +1 @@ +vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing diff --git a/samples/template/scripts/create_template_files.py b/samples/template/scripts/create_template_files.py new file mode 100644 index 00000000..a9122f36 --- /dev/null +++ b/samples/template/scripts/create_template_files.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +""" +Generates a set of template files using structure and defaults of **schema.json + +--output-path Saves it in the specified output directory (/path/to/output). +If the argument is not provided, it will default to the same directory as the script. + +--schema can specify using a specific schema instead of the default of all +""" + +import argparse +import glob +import logging +import os +import pathlib + +from frictionless import Schema + +USAGE = """ +python samples/template/scripts/create_template_files.py --output-dir /path/to/output +""" + + +SAMPLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_REPO_DIR = os.path.dirname(os.path.dirname(SAMPLE_DIR)) +SCHEMAS = glob.glob( + os.path.join(os.path.join(BASE_REPO_DIR, "spec"), "**/*.schema.json"), + recursive=True, +) +DEFAULT_OUT_PATH = "../TIDES" + + +def write_template_for_schema( + schema_filename: str, + out_dir: str, +) -> None: + """Creates blank csvs which comply with a schema. + + Args: + schema_filename (str): Filename with the Frictionless data schema + out_dir (str): Where the csv will be written + """ + _schema = Schema(schema_filename) + fields = [s.name for s in _schema.fields] + _schema_name = pathlib.Path(schema_filename).stem.split(".")[0] + out_filename = os.path.join(out_dir, _schema_name + ".csv") + with open(out_filename, "w") as outfile: + logging.info(f"Writing: {out_filename}") + outfile.write(",".join(fields)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser(description="Generate a template csvs.") + parser.add_argument( + "--schema", + default=SCHEMAS, + help="List of paths to frictionless schemas to generate. Default: all in /spec dir.", + ) + parser.add_argument( + "--output-path", + default=DEFAULT_OUT_PATH, + help="Directory to save the generated files. Default: same directory as script.", + ) + args = parser.parse_args() + + for s in args.schema: + write_template_for_schema(s, args.output_path) diff --git a/samples/template/scripts/requirements.txt b/samples/template/scripts/requirements.txt new file mode 100644 index 00000000..140b95f1 --- /dev/null +++ b/samples/template/scripts/requirements.txt @@ -0,0 +1 @@ +frictionless diff --git a/spec/table-schema.json b/spec/table-schema.json deleted file mode 100644 index a87b49e5..00000000 --- a/spec/table-schema.json +++ /dev/null @@ -1,1676 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with the [Table Schema](/tableschema/) specification.", - "type": [ - "string", - "object" - ], - "required": [ - "fields" - ], - "properties": { - "fields": { - "type": "array", - "minItems": 1, - "items": { - "title": "Table Schema Field", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "String Field", - "description": "The field contains strings, that is, sequences of characters.", - "required": [ - "name" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `string`.", - "enum": [ - "string" - ] - }, - "format": { - "description": "The format keyword options for `string` are `default`, `email`, `uri`, `binary`, and `uuid`.", - "context": "The following `format` options are supported:\n * **default**: any valid string.\n * **email**: A valid email address.\n * **uri**: A valid URI.\n * **binary**: A base64 encoded string representing binary data.\n * **uuid**: A string that is a uuid.", - "enum": [ - "default", - "email", - "uri", - "binary", - "uuid" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `string` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "pattern": { - "type": "string", - "description": "A regular expression pattern to test each value of the property against, where a truthy response indicates validity.", - "context": "Regular expressions `SHOULD` conform to the [XML Schema regular expression syntax](http://www.w3.org/TR/xmlschema-2/#regexs)." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"name\",\n \"type\": \"string\"\n}\n", - "{\n \"name\": \"name\",\n \"type\": \"string\",\n \"format\": \"email\"\n}\n", - "{\n \"name\": \"name\",\n \"type\": \"string\",\n \"constraints\": {\n \"minLength\": 3,\n \"maxLength\": 35\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Number Field", - "description": "The field contains numbers of any kind including decimals.", - "context": "The lexical formatting follows that of decimal in [XMLSchema](https://www.w3.org/TR/xmlschema-2/#decimal): a non-empty finite-length sequence of decimal digits separated by a period as a decimal indicator. An optional leading sign is allowed. If the sign is omitted, '+' is assumed. Leading and trailing zeroes are optional. If the fractional part is zero, the period and following zero(es) can be omitted. For example: '-1.23', '12678967.543233', '+100000.00', '210'.\n\nThe following special string values are permitted (case does not need to be respected):\n - NaN: not a number\n - INF: positive infinity\n - -INF: negative infinity\n\nA number `MAY` also have a trailing:\n - exponent: this `MUST` consist of an E followed by an optional + or - sign followed by one or more decimal digits (0-9)\n - percentage: the percentage sign: `%`. In conversion percentages should be divided by 100.\n\nIf both exponent and percentages are present the percentage `MUST` follow the exponent e.g. '53E10%' (equals 5.3).", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `number`.", - "enum": [ - "number" - ] - }, - "format": { - "description": "There are no format keyword options for `number`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "bareNumber": { - "type": "boolean", - "title": "bareNumber", - "description": "a boolean field with a default of `true`. If `true` the physical contents of this field must follow the formatting constraints already set out. If `false` the contents of this field may contain leading and/or trailing non-numeric characters (which implementors MUST therefore strip). The purpose of `bareNumber` is to allow publishers to publish numeric data that contains trailing characters such as percentages e.g. `95%` or leading characters such as currencies e.g. `€95` or `EUR 95`. Note that it is entirely up to implementors what, if anything, they do with stripped text.", - "default": true - }, - "decimalChar": { - "type": "string", - "description": "A string whose value is used to represent a decimal point within the number. The default value is `.`." - }, - "groupChar": { - "type": "string", - "description": "A string whose value is used to group digits within the number. The default value is `null`. A common value is `,` e.g. '100,000'." - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `number` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "number" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"field-name\",\n \"type\": \"number\"\n}\n", - "{\n \"name\": \"field-name\",\n \"type\": \"number\",\n \"constraints\": {\n \"enum\": [ \"1.00\", \"1.50\", \"2.00\" ]\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Integer Field", - "description": "The field contains integers - that is whole numbers.", - "context": "Integer values are indicated in the standard way for any valid integer.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `integer`.", - "enum": [ - "integer" - ] - }, - "format": { - "description": "There are no format keyword options for `integer`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "bareNumber": { - "type": "boolean", - "title": "bareNumber", - "description": "a boolean field with a default of `true`. If `true` the physical contents of this field must follow the formatting constraints already set out. If `false` the contents of this field may contain leading and/or trailing non-numeric characters (which implementors MUST therefore strip). The purpose of `bareNumber` is to allow publishers to publish numeric data that contains trailing characters such as percentages e.g. `95%` or leading characters such as currencies e.g. `€95` or `EUR 95`. Note that it is entirely up to implementors what, if anything, they do with stripped text.", - "default": true - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `integer` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "integer" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"age\",\n \"type\": \"integer\",\n \"constraints\": {\n \"unique\": true,\n \"minimum\": 100,\n \"maximum\": 9999\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Date Field", - "description": "The field contains temporal date values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `date`.", - "enum": [ - "date" - ] - }, - "format": { - "description": "The format keyword options for `date` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string of YYYY-MM-DD.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `date` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\"\n}\n", - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\",\n \"constraints\": {\n \"minimum\": \"01-01-1900\"\n }\n}\n", - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\",\n \"format\": \"MM-DD-YYYY\"\n}\n" - ] - }, - { - "type": "object", - "title": "Time Field", - "description": "The field contains temporal time values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `time`.", - "enum": [ - "time" - ] - }, - "format": { - "description": "The format keyword options for `time` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string for time.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `time` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"appointment_start\",\n \"type\": \"time\"\n}\n", - "{\n \"name\": \"appointment_start\",\n \"type\": \"time\",\n \"format\": \"any\"\n}\n" - ] - }, - { - "type": "object", - "title": "Date Time Field", - "description": "The field contains temporal datetime values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `datetime`.", - "enum": [ - "datetime" - ] - }, - "format": { - "description": "The format keyword options for `datetime` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string for datetime.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `datetime` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"timestamp\",\n \"type\": \"datetime\"\n}\n", - "{\n \"name\": \"timestamp\",\n \"type\": \"datetime\",\n \"format\": \"default\"\n}\n" - ] - }, - { - "type": "object", - "title": "Year Field", - "description": "A calendar year, being an integer with 4 digits. Equivalent to [gYear in XML Schema](https://www.w3.org/TR/xmlschema-2/#gYear)", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `year`.", - "enum": [ - "year" - ] - }, - "format": { - "description": "There are no format keyword options for `year`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `year` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "integer" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"year\",\n \"type\": \"year\"\n}\n", - "{\n \"name\": \"year\",\n \"type\": \"year\",\n \"constraints\": {\n \"minimum\": 1970,\n \"maximum\": 2003\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Year Month Field", - "description": "A calendar year month, being an integer with 1 or 2 digits. Equivalent to [gYearMonth in XML Schema](https://www.w3.org/TR/xmlschema-2/#gYearMonth)", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `yearmonth`.", - "enum": [ - "yearmonth" - ] - }, - "format": { - "description": "There are no format keyword options for `yearmonth`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `yearmonth` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"month\",\n \"type\": \"yearmonth\"\n}\n", - "{\n \"name\": \"month\",\n \"type\": \"yearmonth\",\n \"constraints\": {\n \"minimum\": 1,\n \"maximum\": 6\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Boolean Field", - "description": "The field contains boolean (true/false) data.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `boolean`.", - "enum": [ - "boolean" - ] - }, - "format": { - "description": "There are no format keyword options for `boolean`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "trueValues": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - }, - "default": [ - "true", - "True", - "TRUE", - "1" - ] - }, - "falseValues": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - }, - "default": [ - "false", - "False", - "FALSE", - "0" - ] - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `boolean` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "boolean" - } - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"registered\",\n \"type\": \"boolean\"\n}\n" - ] - }, - { - "type": "object", - "title": "Object Field", - "description": "The field contains data which can be parsed as a valid JSON object.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `object`.", - "enum": [ - "object" - ] - }, - "format": { - "description": "There are no format keyword options for `object`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply for `object` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"extra\"\n \"type\": \"object\"\n}\n" - ] - }, - { - "type": "object", - "title": "GeoPoint Field", - "description": "The field contains data describing a geographic point.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `geopoint`.", - "enum": [ - "geopoint" - ] - }, - "format": { - "description": "The format keyword options for `geopoint` are `default`,`array`, and `object`.", - "context": "The following `format` options are supported:\n * **default**: A string of the pattern 'lon, lat', where `lon` is the longitude and `lat` is the latitude.\n * **array**: An array of exactly two items, where each item is either a number, or a string parsable as a number, and the first item is `lon` and the second item is `lat`.\n * **object**: A JSON object with exactly two keys, `lat` and `lon`", - "notes": [ - "Implementations `MUST` strip all white space in the default format of `lon, lat`." - ], - "enum": [ - "default", - "array", - "object" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `geopoint` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "array" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"post_office\",\n \"type\": \"geopoint\"\n}\n", - "{\n \"name\": \"post_office\",\n \"type\": \"geopoint\",\n \"format\": \"array\"\n}\n" - ] - }, - { - "type": "object", - "title": "GeoJSON Field", - "description": "The field contains a JSON object according to GeoJSON or TopoJSON", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `geojson`.", - "enum": [ - "geojson" - ] - }, - "format": { - "description": "The format keyword options for `geojson` are `default` and `topojson`.", - "context": "The following `format` options are supported:\n * **default**: A geojson object as per the [GeoJSON spec](http://geojson.org/).\n * **topojson**: A topojson object as per the [TopoJSON spec](https://github.com/topojson/topojson-specification/blob/master/README.md)", - "enum": [ - "default", - "topojson" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `geojson` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"city_limits\",\n \"type\": \"geojson\"\n}\n", - "{\n \"name\": \"city_limits\",\n \"type\": \"geojson\",\n \"format\": \"topojson\"\n}\n" - ] - }, - { - "type": "object", - "title": "Array Field", - "description": "The field contains data which can be parsed as a valid JSON array.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `array`.", - "enum": [ - "array" - ] - }, - "format": { - "description": "There are no format keyword options for `array`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply for `array` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "array" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"options\"\n \"type\": \"array\"\n}\n" - ] - }, - { - "type": "object", - "title": "Duration Field", - "description": "The field contains a duration of time.", - "context": "The lexical representation for duration is the [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) extended format `PnYnMnDTnHnMnS`, where `nY` represents the number of years, `nM` the number of months, `nD` the number of days, 'T' is the date/time separator, `nH` the number of hours, `nM` the number of minutes and `nS` the number of seconds. The number of seconds can include decimal digits to arbitrary precision. Date and time elements including their designator may be omitted if their value is zero, and lower order elements may also be omitted for reduced precision. Here we follow the definition of [XML Schema duration datatype](http://www.w3.org/TR/xmlschema-2/#duration) directly and that definition is implicitly inlined here.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `duration`.", - "enum": [ - "duration" - ] - }, - "format": { - "description": "There are no format keyword options for `duration`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `duration` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"period\"\n \"type\": \"duration\"\n}\n" - ] - }, - { - "type": "object", - "title": "Any Field", - "description": "Any value is accepted, including values that are not captured by the type/format/constraint requirements of the specification.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `any`.", - "enum": [ - "any" - ] - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply to `any` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"notes\",\n \"type\": \"any\"\n" - ] - } - ] - }, - "description": "An `array` of Table Schema Field objects.", - "examples": [ - "{\n \"fields\": [\n {\n \"name\": \"my-field-name\"\n }\n ]\n}\n", - "{\n \"fields\": [\n {\n \"name\": \"my-field-name\",\n \"type\": \"number\"\n },\n {\n \"name\": \"my-field-name-2\",\n \"type\": \"string\",\n \"format\": \"email\"\n }\n ]\n}\n" - ] - }, - "primaryKey": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "string" - } - ], - "description": "A primary key is a field name or an array of field names, whose values `MUST` uniquely identify each row in the table.", - "context": "Field name in the `primaryKey` `MUST` be unique, and `MUST` match a field name in the associated table. It is acceptable to have an array with a single value, indicating that the value of a single field is the primary key.", - "examples": [ - "{\n \"primaryKey\": [\n \"name\"\n ]\n}\n", - "{\n \"primaryKey\": [\n \"first_name\",\n \"last_name\"\n ]\n}\n" - ] - }, - "foreignKeys": { - "type": "array", - "minItems": 1, - "items": { - "title": "Table Schema Foreign Key", - "description": "Table Schema Foreign Key", - "type": "object", - "required": [ - "fields", - "reference" - ], - "oneOf": [ - { - "properties": { - "fields": { - "type": "array", - "items": { - "type": "string", - "minItems": 1, - "uniqueItems": true, - "description": "Fields that make up the primary key." - } - }, - "reference": { - "type": "object", - "required": [ - "resource", - "fields" - ], - "properties": { - "resource": { - "type": "string", - "default": "" - }, - "fields": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - } - } - } - }, - { - "properties": { - "fields": { - "type": "string", - "description": "Fields that make up the primary key." - }, - "reference": { - "type": "object", - "required": [ - "resource", - "fields" - ], - "properties": { - "resource": { - "type": "string", - "default": "" - }, - "fields": { - "type": "string" - } - } - } - } - } - ] - }, - "examples": [ - "{\n \"foreignKeys\": [\n {\n \"fields\": \"state\",\n \"reference\": {\n \"resource\": \"the-resource\",\n \"fields\": \"state_id\"\n }\n }\n ]\n}\n", - "{\n \"foreignKeys\": [\n {\n \"fields\": \"state\",\n \"reference\": {\n \"resource\": \"\",\n \"fields\": \"id\"\n }\n }\n ]\n}\n" - ] - }, - "missingValues": { - "type": "array", - "items": { - "type": "string" - }, - "default": [ - "" - ], - "description": "Values that when encountered in the source, should be considered as `null`, 'not present', or 'blank' values.", - "context": "Many datasets arrive with missing data values, either because a value was not collected or it never existed.\nMissing values may be indicated simply by the value being empty in other cases a special value may have been used e.g. `-`, `NaN`, `0`, `-9999` etc.\nThe `missingValues` property provides a way to indicate that these values should be interpreted as equivalent to null.\n\n`missingValues` are strings rather than being the data type of the particular field. This allows for comparison prior to casting and for fields to have missing value which are not of their type, for example a `number` field to have missing values indicated by `-`.\n\nThe default value of `missingValue` for a non-string type field is the empty string `''`. For string type fields there is no default for `missingValue` (for string fields the empty string `''` is a valid value and need not indicate null).", - "examples": [ - "{\n \"missingValues\": [\n \"-\",\n \"NaN\",\n \"\"\n ]\n}\n", - "{\n \"missingValues\": []\n}\n" - ] - } - }, - "examples": [ - "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" - ] -} \ No newline at end of file diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json deleted file mode 100644 index 8d055dd5..00000000 --- a/spec/tides-data-package.json +++ /dev/null @@ -1,688 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TIDES Data Package", - "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", - "type": "object", - "required": [ - "profile", - "resources", - "title" - ], - "properties": { - "profile": { - "enum": [ - "tides-data-package" - ], - "propertyOrder": 10, - "title": "Profile", - "description": "The profile of this descriptor.", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string", - "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" - ] - }, - "title": { - "propertyOrder": 20, - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "name": { - "propertyOrder": 30, - "title": "Name", - "description": "Short sluggable (https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "{\n \"name\": \"tides-data-package\"\n}\n" - ] - }, - "description": { - "propertyOrder": 40, - "format": "textarea", - "title": "Description", - "description": "Short description of TIDES data package.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "agency": { - "propertyOrder": 50, - "format": "textarea", - "title": "Agency", - "description": "Transit agency name.", - "type": "string", - "examples": [ - "{\n \"agency\": \"Coos County Area Transit\"\n}\n" - ] - }, - "ntd_id": { - "propertyOrder": 60, - "title": "NTD ID", - "description": "ID for the National Transit Database.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "Background on NTD available at https://www.transit.dot.gov/ntd", - "examples": [ - "{\n \"ntd_id\": \"0R02-00307\"\n}\n" - ] - }, - "contributors": { - "propertyOrder": 70, - "title": "Contributors", - "description": "Array of data contributors `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", - "type": "array", - "minItems": 1, - "items": { - "title": "Contributor", - "description": "A contributor to this descriptor.", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "{\n \"title\": \"Frank Julian Sprague\"\n}\n" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this contributor.", - "type": "string" - }, - "role": { - "title": "Role", - "description": "The contributor's role in the project", - "type": "string", - "default": "contributor" - } - }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." - }, - "examples": [ - "{\n \"contributors\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"contributors\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" - ] - }, - "maintainers": { - "propertyOrder": 80, - "title": "Maintainers", - "description": "Array of data maintainers `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", - "type": "array", - "minItems": 1, - "items": { - "title": "Maintainer", - "description": "A maintainer of this descriptor.", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "{\n \"name\": \"George Francis Train\"\n}\n" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this maintainer.", - "type": "string" - }, - "role": { - "title": "Role", - "description": "The maintainer's role in the project", - "type": "string", - "default": "maintainer" - } - }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." - }, - "examples": [ - "{\n \"maintainers\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"maintainers\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" - ] - }, - "licenses": { - "propertyOrder": 90, - "title": "Licenses", - "description": "The license(s) under which this package is published.", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "A license for this descriptor.", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a Unix-style relative file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/gpl-2-0/\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - } - } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" - ] - }, - "resources": { - "propertyOrder": 100, - "title": "Tabular Data Resources", - "description": "Array of data files included in your package, formated as a tabular-data-resource", - "type": "array", - "minItems": 1, - "items": { - "title": "Tabular Data Resource", - "description": "A Tabular Data Resource.", - "type": "object", - "required": [ - "name", - "path", - "schema" - ], - "properties": { - "profile": { - "enum": [ - "tabular-data-resource" - ], - "propertyOrder": 10, - "title": "Profile", - "description": "Should be `tides-data-package`", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string", - "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" - ] - }, - "name": { - "propertyOrder": 20, - "title": "Name", - "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "{\n \"name\": \"tabular-data-resource\"\n}\n" - ] - }, - "path": { - "propertyOrder": 30, - "title": "Path", - "description": "A reference to the data for this resource, as a valid URI string.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "schema": { - "propertyOrder": 40, - "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with one of the [TIDES](/) table specifications.", - "type": "string", - "required": [ - "name", - "path" - ], - "properties": { - "name": { - "propertyOrder": 10, - "title": "Name", - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "examples": [ - "{\n \"name\": \"fare_transactions\"\n}\n", - "{\n \"name\": \"train_cars\"\n}\n" - ] - }, - "path": { - "propertyOrder": 20, - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/fare_transactions.schema.json\"\n}\n", - "{\n \"path\": \"https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json\"\n}\n" - ] - } - }, - "examples": [ - "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" - ] - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ], - "propertyOrder": 50 - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ], - "propertyOrder": 60, - "format": "textarea" - }, - "homepage": { - "propertyOrder": 70, - "title": "Home Page", - "description": "The home on the web that is related to this data package.", - "type": "string", - "format": "uri", - "examples": [ - "{\n \"homepage\": \"http://example.com/\"\n}\n" - ] - }, - "sources": { - "propertyOrder": 140, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "The raw sources for this resource.", - "type": "array", - "minItems": 0, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "Description of the data source.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "component": { - "title": "Component", - "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", - "type": "string" - }, - "product": { - "title": "Product", - "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string" - }, - "product_version": { - "title": "Product Version", - "description": "Describe the version of the product was used.", - "type": "string" - }, - "vendor": { - "title": "Vendor", - "description": "What company makes this product?", - "type": "string" - } - } - } - }, - "licenses": { - "description": "Should be `[{\"name\": \"Apache-2.0\"}]` to be consistent with this repository", - "propertyOrder": 150, - "options": { - "hidden": true - }, - "title": "Licenses", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "A license for this descriptor.", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/apache-2-0/\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - } - } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" - ] - }, - "dialect": { - "propertyOrder": 50, - "title": "CSV Dialect", - "description": "The CSV dialect descriptor.", - "type": [ - "string", - "object" - ], - "required": [ - "delimiter", - "doubleQuote" - ], - "properties": { - "csvddfVersion": { - "title": "CSV Dialect schema version", - "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", - "type": "number", - "default": 1.2, - "examples:": [ - "{\n \"csvddfVersion\": \"1.2\"\n}\n" - ] - }, - "delimiter": { - "title": "Delimiter", - "description": "A character sequence to use as the field separator.", - "type": "string", - "default": ",", - "examples": [ - "{\n \"delimiter\": \",\"\n}\n", - "{\n \"delimiter\": \";\"\n}\n" - ] - }, - "doubleQuote": { - "title": "Double Quote", - "description": "Specifies the handling of quotes inside fields.", - "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", - "type": "boolean", - "default": true, - "examples": [ - "{\n \"doubleQuote\": true\n}\n" - ] - }, - "lineTerminator": { - "title": "Line Terminator", - "description": "Specifies the character sequence that must be used to terminate rows.", - "type": "string", - "default": "\r\n", - "examples": [ - "{\n \"lineTerminator\": \"\\r\\n\"\n}\n", - "{\n \"lineTerminator\": \"\\n\"\n}\n" - ] - }, - "nullSequence": { - "title": "Null Sequence", - "description": "Specifies the null sequence, for example, `\\N`.", - "type": "string", - "examples": [ - "{\n \"nullSequence\": \"\\N\"\n}\n" - ] - }, - "quoteChar": { - "title": "Quote Character", - "description": "Specifies a one-character string to use as the quoting character.", - "type": "string", - "default": "\"", - "examples": [ - "{\n \"quoteChar\": \"'\"\n}\n" - ] - }, - "escapeChar": { - "title": "Escape Character", - "description": "Specifies a one-character string to use as the escape character.", - "type": "string", - "examples": [ - "{\n \"escapeChar\": \"\\\\\"\n}\n" - ] - }, - "skipInitialSpace": { - "title": "Skip Initial Space", - "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", - "type": "boolean", - "default": false, - "examples": [ - "{\n \"skipInitialSpace\": true\n}\n" - ] - }, - "header": { - "title": "Header", - "description": "Specifies if the file includes a header row, always as the first row in the file.", - "type": "boolean", - "default": true, - "examples": [ - "{\n \"header\": true\n}\n" - ] - }, - "commentChar": { - "title": "Comment Character", - "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", - "type": "string", - "examples": [ - "{\n \"commentChar\": \"#\"\n}\n" - ] - }, - "caseSensitiveHeader": { - "title": "Case Sensitive Header", - "description": "Specifies if the case of headers is meaningful.", - "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", - "type": "boolean", - "default": false, - "examples": [ - "{\n \"caseSensitiveHeader\": true\n}\n" - ] - } - }, - "examples": [ - "{\n \"dialect\": {\n \"delimiter\": \";\"\n }\n}\n", - "{\n \"dialect\": {\n \"delimiter\": \"\\t\",\n \"quoteChar\": \"'\",\n \"commentChar\": \"#\"\n }\n}\n" - ] - }, - "format": { - "propertyOrder": 80, - "title": "Format", - "description": "The file format of this resource.", - "context": "`csv`, `xls`, `json` are examples of common formats.", - "type": "string", - "examples": [ - "{\n \"format\": \"xls\"\n}\n" - ] - }, - "mediatype": { - "propertyOrder": 90, - "title": "Media Type", - "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", - "type": "string", - "pattern": "^(.+)/(.+)$", - "examples": [ - "{\n \"mediatype\": \"text/csv\"\n}\n" - ] - }, - "encoding": { - "propertyOrder": 100, - "title": "Encoding", - "description": "The file encoding of this resource.", - "type": "string", - "default": "utf-8", - "examples": [ - "{\n \"encoding\": \"utf-8\"\n}\n" - ] - }, - "bytes": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Bytes", - "description": "The size of this resource in bytes.", - "type": "integer", - "examples": [ - "{\n \"bytes\": 2082\n}\n" - ] - }, - "hash": { - "propertyOrder": 120, - "options": { - "hidden": true - }, - "title": "Hash", - "type": "string", - "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", - "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", - "examples": [ - "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", - "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" - ] - } - } - }, - "examples": [ - "{\n \"resources\": [\n {\n \"name\": \"my-data\",\n \"data\": [\n \"data.csv\"\n ],\n \"schema\": \"tableschema.json\",\n \"mediatype\": \"text/csv\"\n }\n ]\n}\n" - ] - }, - "sources": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "Array of data sources formatted as a source. Recommended to be documented either here at the top-level or for each individual resource", - "type": "array", - "minItems": 0, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "path": { - "title": "Path", - "description": "A fully qualified URL, or a POSIX file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - } - } - }, - "examples": [ - "{\n \"sources\": [\n {\n \"title\": \"World Bank and OECD\",\n \"path\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" - ] - } - } - } diff --git a/spec/tides-datapackage-profile.json b/spec/tides-datapackage-profile.json new file mode 100644 index 00000000..aa25313b --- /dev/null +++ b/spec/tides-datapackage-profile.json @@ -0,0 +1,690 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TIDES Data Package", + "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", + "type": "object", + "required": [ + "title", + "profile", + "resources" + + ], + "recommended":[ + "name", + "description", + "agency", + "ntd_id", + "licenses", + "contributors", + "maintainers" + ], + "properties": { + "profile": { + "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", + "propertyOrder": 10, + "title": "Profile", + "description": "The json-schema profile used to validate this datapackage descriptor. Should be `https://raw.githubusercontent.com/TIDES-transit/TIDES//spec/tides-datapackage-profile.json`. Defaults to `https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`", + "format": "uri", + "examples": [ + "`https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`" + ] + }, + "title": { + "propertyOrder": 20, + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "Make Believe Trolley 1/1/2001-2/1/2001" + ] + }, + "name": { + "propertyOrder": 30, + "title": "Name", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "trolley_2001_1" + ] + }, + "description": { + "propertyOrder": 40, + "format": "textarea", + "title": "Description", + "description": "Short description of TIDES data package.", + "type": "string", + "examples": [ + "Raw and processed data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" + ] + }, + "agency": { + "propertyOrder": 50, + "format": "textarea", + "title": "Agency", + "description": "Transit agency name.", + "type": "string", + "examples": [ + "Make Believe Neighborhood Trolley" + ] + }, + "ntd_id": { + "propertyOrder": 60, + "title": "NTD ID", + "description": "ID for the National Transit Database.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "Background on NTD available at https://www.transit.dot.gov/ntd", + "examples": [ + "0R02-00307" + ] + }, + "contributors": { + "propertyOrder": 70, + "title": "Contributors", + "description": "Array of data contributors.", + "type": "array", + "minItems": 1, + "items": { + "title": "Contributor", + "description": "A contributor to this descriptor.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "Daniel Tiger" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "daniel@makebelievetrolley.gov" + ] + }, + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this contributor.", + "type": "string", + "example": "Make Believe Neighborhood Trolley" + }, + "role": { + "title": "Role", + "description": "The contributor's role in the project", + "type": "string", + "default": "contributor" + } + }, + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." + }, + "examples": [ + "{'name':'Daniel Tiger','email': 'daniel@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'contributor'}" + ] + }, + "maintainers": { + "propertyOrder": 80, + "title": "Maintainers", + "description": "Array of data maintainers", + "type": "array", + "minItems": 1, + "items": { + "title": "Maintainer", + "description": "A maintainer of this descriptor.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "Henrietta Pussycat" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "henrietta@makebelievetrolley.gov" + ] + }, + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this maintainer.", + "type": "string" + }, + "role": { + "title": "Role", + "description": "The maintainer's role in the project", + "type": "string", + "default": "maintainer" + } + }, + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." + }, + "examples": [ + "{'name':'Henrietta Pussycat','email': 'henrietta@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'maintainer'}" + + ] + }, + "licenses": { + "propertyOrder": 90, + "title": "Licenses", + "description": "The license(s) under which this package is published.", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", + "type": "object", + "anyOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "path" + ] + } + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] + }, + "path": { + "title": "Path", + "description": "A fully qualified public URL, or a Unix-style relative file path.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "https://opendefinition.org/licenses/odc-by/", + "license.md" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "title": { + "title": "Title", + "description": "A human-readable title of the license.", + "type": "string", + "examples": [ + "Creative Commons Attribution 4.0" + ] + } + } + }, + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" + ] + }, + "resources": { + "propertyOrder": 100, + "title": "Tabular Data Resources", + "description": "Array describing the data files included in your package, formated as a [`tabular-data-resource`](#tabular-data-resource)", + "type": "array", + "minItems": 1, + "items": { + "title": "Tabular Data Resource", + "description": "A Tabular Data Resource.", + "type": "object", + "required": [ + "name", + "path", + "schema", + "profile" + ], + "recommended": [ + "title", + "description", + "sources", + "licenses" + ], + "properties": { + "profile": { + "default": "tabular-data-resource", + "propertyOrder": 10, + "title": "Profile", + "description": "Must be `tabular-data-resource`", + "type": "string", + "examples": [ + "tabular-data-resource" + ] + }, + "name": { + "propertyOrder": 20, + "title": "Name", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string for a data table.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "vehicle_locations_2001_01" + ] + }, + "path": { + "propertyOrder": 30, + "title": "Path", + "description": "A reference to the data for this resource, as a valid URI string.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "event/2001/vehicle_locations_2001_01.csv" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "tides-table": { + "enum": [ + "devices", + "fare_transactions", + "operators", + "station_activities", + "passenger_events", + "stop_visits", + "train_cars", + "trips_performed", + "vehicle_locations", + "vehicle_train_cars", + "vehicles" + ], + "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "type": "string", + "examples": [ + "fare_transactions", + "train_cars" + ] + }, + "schema": { + "title": "Table Schema", + "description": "Location of the table schema for this resource.", + "type": "string", + "examples": [ + "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json" + ] + }, + "title": { + "title": "Title", + "description": "A human-readable title for the data resource.", + "type": "string", + "examples": [ + "Neighborhood Trolley Vehicle Locations: 1/1/2001-2/1/2001" + ], + "propertyOrder": 50 + }, + "description": { + "title": "Description", + "description": "D", + "type": "string", + "examples": [ + "Vehicle location event data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" + ], + "propertyOrder": 60, + "format": "textarea" + }, + "homepage": { + "propertyOrder": 70, + "title": "Home Page", + "description": "The home on the web that is related to this data package.", + "type": "string", + "format": "uri", + "examples": [ + "http://example.com" + ] + }, + "sources": { + "propertyOrder": 140, + "options": { + "hidden": true + }, + "title": "Sources", + "description": "The raw sources for this resource which describe where the data came from.", + "type": "array", + "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], + "items": { + "title": "Source", + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "title": "Title", + "description": "Human-readable title of the data source.", + "type": "string", + "examples": [ + "Trolley CAD/AVL System" + ] + }, + "component": { + "title": "Component", + "description": "The technology component used to generate this data (directly or indirectly)", + "type": "string", + "examples": [ + "CAD/AVL", + "AFC", + "APC" + ] + }, + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", + "type": "string", + "examples": [ + "TrolleyMaster" + ] + }, + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" + }, + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" + } + } + } + }, + "licenses": { + "propertyOrder": 90, + "title": "Licenses", + "description": "The license(s) under which this package is published.", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", + "type": "object", + "anyOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "path" + ] + } + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] + }, + "path": { + "title": "Path", + "description": "A fully qualified public URL, or a Unix-style relative file path.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "https://opendefinition.org/licenses/odc-by/", + "license.md" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "title": { + "title": "Title", + "description": "A human-readable title of the license.", + "type": "string", + "examples": [ + "Creative Commons Attribution 4.0" + ] + } + } + }, + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" + ] + }, + "dialect": { + "propertyOrder": 50, + "title": "CSV Dialect", + "description": "The CSV dialect descriptor.", + "type": [ + "string", + "object" + ], + "required": [ + "delimiter", + "doubleQuote" + ], + "properties": { + "csvddfVersion": { + "title": "CSV Dialect schema version", + "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", + "type": "number", + "default": 1.2, + "examples:": [ + "1.2" + ] + }, + "delimiter": { + "title": "Delimiter", + "description": "A character sequence to use as the field separator.", + "type": "string", + "default": ",", + "examples": [ + "`;`", + "`|`" + ] + }, + "doubleQuote": { + "title": "Double Quote", + "description": "Specifies the handling of quotes inside fields.", + "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", + "type": "boolean", + "default": true + }, + "lineTerminator": { + "title": "Line Terminator", + "description": "Specifies the character sequence that must be used to terminate rows.", + "type": "string", + "default": "\r\n", + "examples": [ + "`\r`", + "`\n`" + ] + }, + "nullSequence": { + "title": "Null Sequence", + "description": "Specifies the null sequence, for example, `\\N`.", + "type": "string", + "examples": [ + "`\\N`" + ] + }, + "quoteChar": { + "title": "Quote Character", + "description": "Specifies a one-character string to use as the quoting character.", + "type": "string", + "default": "\"", + "examples": [ + "`'`" + ] + }, + "escapeChar": { + "title": "Escape Character", + "description": "Specifies a one-character string to use as the escape character.", + "type": "string", + "examples": [ + "`\\`" + ] + }, + "skipInitialSpace": { + "title": "Skip Initial Space", + "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", + "type": "boolean", + "default": false + }, + "header": { + "title": "Header", + "description": "Specifies if the file includes a header row, always as the first row in the file.", + "type": "boolean", + "default": true + }, + "commentChar": { + "title": "Comment Character", + "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", + "type": "string", + "examples": [ + "`#`" + ] + }, + "caseSensitiveHeader": { + "title": "Case Sensitive Header", + "description": "Specifies if the case of headers is meaningful.", + "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", + "type": "boolean", + "default": false + } + }, + "examples": [ + "{'delimiter': ';' }" + ] + }, + "format": { + "propertyOrder": 80, + "title": "Format", + "description": "The file format of this resource.", + "context": "`csv`, `xls`, `json` are examples of common formats.", + "type": "string", + "examples": [ + "`xls`" + ] + }, + "mediatype": { + "propertyOrder": 90, + "title": "Media Type", + "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", + "type": "string", + "pattern": "^(.+)/(.+)$", + "examples": [ + "`text/csv`" + ] + }, + "encoding": { + "propertyOrder": 100, + "title": "Encoding", + "description": "The file encoding of this resource.", + "type": "string", + "default": "utf-8", + "examples": [ + "`utf-8`" + ] + }, + "bytes": { + "propertyOrder": 110, + "options": { + "hidden": true + }, + "title": "Bytes", + "description": "The size of this resource in bytes.", + "type": "integer", + "examples": [ + "2082" + ] + }, + "hash": { + "propertyOrder": 120, + "options": { + "hidden": true + }, + "title": "Hash", + "type": "string", + "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", + "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", + "examples": [ + "d25c9c77f588f5dc32059d2da1136c02", + "SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0" + ] + } + } + } + }, + "sources": { + "propertyOrder": 140, + "options": { + "hidden": true + }, + "title": "Sources", + "description": "The raw sources for this data package which describe where the data came from. Can be alternatively specified at the individual resource-level.", + "type": "array", + "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], + "items": { + "title": "Source", + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "title": "Title", + "description": "Human-readable title of the data source.", + "type": "string", + "examples": [ + "Trolley CAD/AVL System" + ] + }, + "component": { + "title": "Component", + "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", + "type": "string", + "examples": [ + "CAD/AVL", + "AFC", + "APC" + ] + }, + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", + "type": "string", + "examples": [ + "TrolleyMaster" + ] + }, + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" + }, + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" + } + } + } + } + } +} diff --git a/tests/test_all b/tests/test_all deleted file mode 100755 index cfd886da..00000000 --- a/tests/test_all +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Runs pre-commit, then all the scripts in this dir, and then builds documentaiton locally - -# Get the list of script files in the directory -script_files=("$(dirname "$(realpath "$0")")"/*) -num_files=$((${#script_files[@]} + 1)) -test_num=1 -error_scripts=() - -echo "Found $num_files tests" - -echo "----- Running test 1 of $num_files: pre-commit" -pre-commit run --all-files - -# Iterate over script files in the directory -for script_file in "${script_files[@]}"; do - - # Exclude the current script and non-executable files from execution - if [ "$(basename "$script_file")" != "test_all" ]; then - ((test_num++)) - if [ -x "$script_file" ]; then - # Update progress - echo "----- Running test $test_num of $num_files: $script_file" - - # Execute the script and capture errors if there are any - if output=$("$script_file" 2>&1); then - echo ":-) $script_file ran successfully." - else - echo "!!! $script_file encountered an error." - error_scripts+=("$script_file") - echo "$output" >&2 - fi - else - # Notify non-executable files - echo "!!! File is not executable so was not run: $script_file." - echo - fi - fi -done -echo "----- Running test $num_files/$num_files: Building documentation locally" -mkdocs build --quiet - -if [ ${#error_scripts[@]} -gt 0 ]; then - echo "The following scripts executed with errors:" - for script in "${error_scripts[@]}"; do - echo "$script" - done -fi diff --git a/tests/test_local_spec b/tests/test_local_spec new file mode 100755 index 00000000..195bf940 --- /dev/null +++ b/tests/test_local_spec @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +description="Validates the local spec and documentation." +usage="tests/test_local_spec" + +################################################################################ +# Run Pre-Commit # +################################################################################ +echo "Running pre-commit." +for _ in {1..2}; do + if output=$(pre-commit run --all-files 2>&1); then + echo ":-) pre-commit ran successfully on try $_." + break + fi +done + +################################################################################ +# Run Tests # +################################################################################ +echo "Running tests for local compatibility." + +# Function to execute the script and capture errors if there are any +execute_script() { + script=$1 + shift + echo "--- Running $script $@ ---" + if output=$("$script" "$@" 2>&1); then + echo ":-) $script ran successfully." + else + echo "\033[31m!!! $script encountered an error.\033[0m" + echo "$output" >&2 + error_scripts+=("$script") + fi +} + +error_scripts=() +# Execute the scripts +execute_script "tests/validate_profile" +execute_script "tests/validate_table_schemas" + +################################################################################ +# Build Docs Locally # +################################################################################ +echo "Building docs locally." +if output=$(mkdocs build --quiet 2>&1); then + echo ":-) Docs built successfully." +else + echo "\033[31m!!! mkdocs encountered an error.\033[0m" + echo "$output" >&2 + error_scripts+=("mkdocs build --quiet") +fi + +################################################################################ +# Summarize Errors # +################################################################################ +if [ ${#error_scripts[@]} -gt 0 ]; then + echo "The following scripts executed with errors:" + for script in "${error_scripts[@]}"; do + echo "$script" + done +fi diff --git a/tests/test_samples_to_canonical b/tests/test_samples_to_canonical new file mode 100755 index 00000000..0bfc5ac9 --- /dev/null +++ b/tests/test_samples_to_canonical @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +description="Run test to validate local samples to canonical remote spec" +usage="tests/test_samples_to_canonical" + +################################################################################ +# Specify Scripts and Parameters # +################################################################################ +test_dir=$(dirname "$0") +script="$test_dir/validate_samples" +args="-R" + +################################################################################ +# Run Test # +################################################################################ + +echo "Running the test to validate local samples to canonical remote spec" + +# Execute the script and capture errors if there are any +if output=$("$script" $args 2>&1); then + echo ":-) $script ran successfully." +else + printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" + echo "$output" >&2 +fi \ No newline at end of file diff --git a/tests/test_samples_to_local b/tests/test_samples_to_local new file mode 100755 index 00000000..57041bf8 --- /dev/null +++ b/tests/test_samples_to_local @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +description="Run test to validate local samples to local remote spec" +usage="tests/test_samples_to_local" + +################################################################################ +# Specify Scripts and Parameters # +################################################################################ +test_dir=$(dirname "$0") +script="$test_dir/validate_samples" +args="-L" + +################################################################################ +# Run Test # +################################################################################ + +echo "Running the test to validate local samples to local spec" + +# Execute the script and capture errors if there are any +if output=$("$script" $args 2>&1); then + echo ":-) $script ran successfully." +else + printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" + echo "$output" >&2 +fi \ No newline at end of file diff --git a/tests/validate_profile b/tests/validate_profile index 81f4ddd1..2c200be0 100755 --- a/tests/validate_profile +++ b/tests/validate_profile @@ -1,17 +1,24 @@ #!/usr/bin/env bash -echo "Validating tides-data-package.json profile" +# Validate "tides-datapackage-profile.json" against the Frictionless Data Package schema. +# The script downloads the Data Packaage schema file to a temporary directory using curl and +# then uses the jsonschema-cli tool to perform the validation. + +# Usage: ./tests/validate_profile +# Note: should be run from the base directory of the repo + +echo "--- Validating tides-datapackage-profile.json ---" # Set the temporary directory path temp_dir=$(mktemp -d) -# Download the schema file to the temporary directory +# Download the data package schema file to the temporary directory schema_url="https://specs.frictionlessdata.io/schemas/data-package.json" schema_file="$temp_dir/data-package.json" curl -o "$schema_file" "$schema_url" -# Validate datapackage against the downloaded schema -datapackage_file="spec/tides-data.json" +echo "Validate tides-datapackage-profile against the downloaded schema" +datapackage_file="spec/tides-datapackage-profile.json" jsonschema-cli validate "$schema_file" "$datapackage_file" # Clean up the temporary directory diff --git a/tests/validate_samples b/tests/validate_samples index cac8c226..c628838a 100755 --- a/tests/validate_samples +++ b/tests/validate_samples @@ -1,8 +1,100 @@ #!/usr/bin/env bash -echo "Validating sample data" + +description="Validate all the TIDES data in the samples directory" + +usage=" +Usage: bin/validate_samples.sh [-R] [-L] [-r remote_spec_ref] [-l local_spec_ref] + -R: Optional. Use the default remote profile ref (DEFAULT_REMOTE_REF, which is set to "main"). + -L: Optional. Use the default local spec path (DEFAULT_LOCAL_SPEC), which is set to "spec".. + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile where the profile is in the sub-path /spec/tides-data-package.json. + Should not be used with -l option. Example: -r main. + -l local_spec_path: Optional. Specify the location of the local tides-data-package-json to use. + Should not be used with -r option. Example: -l spec. +" + +example_usage="tests/validate_samples -r main" + +################################################################################ +# Default Parameters # +################################################################################ +DEFAULT_REMOTE_REF="main" +DEFAULT_LOCAL_SPEC="spec" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# Source utility functions # +################################################################################ + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bin_path="$script_dir/../bin" + +################################################################################ +# Process the input options. # +################################################################################ +remote_spec_ref="" +local_spec_path="" +use_default_remote=false +use_default_local=false + +# Parse command-line arguments +while getopts "RLr:l:" opt; do + case $opt in + R) + use_default_remote=true + ;; + L) + use_default_local=true + ;; + r) + remote_spec_ref="$OPTARG" + ;; + l) + local_spec_path="$OPTARG" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +############################################################################### +# Main # +################################################################################ + +echo "$description" + +# Set default values if requested +if [ "$use_default_local" = true ]; then + local_spec_path="$DEFAULT_LOCAL_SPEC" +fi + +if [ "$use_default_remote" = true ]; then + remote_spec_ref="$DEFAULT_REMOTE_REF" +fi + shopt -s nullglob for file in samples/*/TIDES/datapackage.json; do - echo "Validating: $file" - frictionless validate --schema-sync $file + sample_dir=$(echo "$file" | sed 's|/datapackage.json$||') + echo "...validating $sample_dir" + "$bin_path/validate-datapackage" -d "$sample_dir" -r "$remote_spec_ref" -l "$local_spec_path" done shopt -u nullglob