From f026881490f3c13ef7861a5c9cb49a1eb79c334b Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 14 Jul 2023 14:52:05 -0700 Subject: [PATCH 01/75] Add sample data structure, local validation scripts, and documentation Also - update errors/clarity in datapackage.json template --- README.md | 31 + bin/validate-data-package | 68 + bin/validate-data-package-json | 64 + docs/datapackage.md | 59 + docs/samples.md | 9 + main.py | 186 ++- mkdocs.yml | 9 + samples/README.md | 49 + samples/template/README.md | 16 + samples/template/TIDES/datapackage.json | 171 +++ samples/template/TIDES/devices.csv | 1 + samples/template/TIDES/fare_transactions.csv | 1 + samples/template/TIDES/operators.csv | 1 + samples/template/TIDES/passenger_events.csv | 1 + samples/template/TIDES/station_activities.csv | 1 + samples/template/TIDES/stop_visits.csv | 1 + samples/template/TIDES/train_cars.csv | 1 + samples/template/TIDES/trips_performed.csv | 1 + samples/template/TIDES/vehicle_locations.csv | 1 + samples/template/TIDES/vehicle_train_cars.csv | 1 + samples/template/TIDES/vehicles.csv | 1 + .../template/scripts/create_template_files.py | 69 + samples/template/scripts/requirements.txt | 1 + spec/tides-data-package.json | 1296 +++++++++-------- 24 files changed, 1393 insertions(+), 646 deletions(-) create mode 100644 bin/validate-data-package create mode 100644 bin/validate-data-package-json create mode 100644 docs/datapackage.md create mode 100644 docs/samples.md create mode 100644 samples/README.md create mode 100644 samples/template/README.md create mode 100644 samples/template/TIDES/datapackage.json create mode 100644 samples/template/TIDES/devices.csv create mode 100644 samples/template/TIDES/fare_transactions.csv create mode 100644 samples/template/TIDES/operators.csv create mode 100644 samples/template/TIDES/passenger_events.csv create mode 100644 samples/template/TIDES/station_activities.csv create mode 100644 samples/template/TIDES/stop_visits.csv create mode 100644 samples/template/TIDES/train_cars.csv create mode 100644 samples/template/TIDES/trips_performed.csv create mode 100644 samples/template/TIDES/vehicle_locations.csv create mode 100644 samples/template/TIDES/vehicle_train_cars.csv create mode 100644 samples/template/TIDES/vehicles.csv create mode 100644 samples/template/scripts/create_template_files.py create mode 100644 samples/template/scripts/requirements.txt diff --git a/README.md b/README.md index 62de1b4b..a6e372ec 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,37 @@ Human-friendlier documentation is auto-generated and available at: - [Architecture](https://tides-transit.github.io/TIDES/main/architecture) - [Table Schemas](https://tides-transit.github.io/TIDES/main/tables) +## Example Data + +Sample data can be found in the `/samples` directory, with one directory for each example. + +## Validating TIDES data + +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Data Package + +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/datapackage.json). + +Once this is created, mapping the data files to the schema, simply run: + +```sh +frictionless validate datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +``` + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/bin/validate-data-package b/bin/validate-data-package new file mode 100644 index 00000000..7f95647e --- /dev/null +++ b/bin/validate-data-package @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Script: validate_frictionless_package.sh +# Description: Bash script to validate a Frictionless Data Package using the Frictionless CLI. +# Usage: validate_frictionless_package.sh [-v tides_version | -l local_schema_location] [-d dataset_location] +# -v tides_version: Optional. Specify the version of the TIDES specification or 'local' to +# use a local schema. Default is to use the schema specified in the datapackage. +# -l local_schema_location: Optional. Specify the location of the local schema directory. +# Default is '../spec'. Is only used if tides_version = local. +# -d dataset_location: Optional. Specify the location of the TIDES datapackage.json. +# Default is the current directory. + +# Set default values +tides_version="" +local_schema_location="../spec" +dataset_location="." + +# Parse command-line arguments +while getopts ":v:l:d:" opt; do + case $opt in + v) + tides_version=$OPTARG + ;; + l) + local_schema_location=$OPTARG + ;; + d) + dataset_location=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +# Create a temporary data package if using a different schema reference or a local schema +tmp_datapackage="" +if [ "$tides_version" != "" ] then + tmp_datapackage=$(mktemp) + cp "$dataset_location/datapackage.json" "$tmp_datapackage" +fi + +# Set the schema URL based on the option chosen +schema_url="" +if [ "$tides_version" == "local" ]; then + schema_path_prefix="$local_schema_location" +else + schema_path_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$tides_version/spec" +fi + +# Update the 'schema' property in the temporary copy of the datapackage.json file, if applicable +if [ "$tmp_datapackage" != "" ]; then + schema_file=$(echo "$tmp_datapackage" | sed 's/\//\\\//g') + sed -E -i "s/\"schema\": \"[^\/]+\.schema\.json\"/\"schema\": \"$schema_path_prefix\/\${schema_file##*\/}\"/g" "$tmp_datapackage" + dataset_location="$tmp_datapackage" +fi + +# Validate the data package JSON against the TIDES schema +./validate-data-package-json.sh -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" + +# Validate the Frictionless Data Package using the Frictionless CLI +frictionless validate "$dataset_location" + +# Remove the temporary data package file, if applicable +if [ "$tmp_datapackage" != "" ]; then + rm "$tmp_datapackage" +fi diff --git a/bin/validate-data-package-json b/bin/validate-data-package-json new file mode 100644 index 00000000..504e08ca --- /dev/null +++ b/bin/validate-data-package-json @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Script to validate a local JSON file against a schema specified in a GitHub repository. +# Usage: validate-data-package-json.sh [-r ref | -l local_schema_location] [-f datapackage_file] +# -r ref: Optional. Specify the ref name of the GitHub repository. Default is 'main'. +# -l local_schema_location: Optional. Specify the location of the local schema directory. +# -f datapackage_file: Optional. Specify the location of the datapackage.json file. Default is 'datapackage.json' in the execution directory. + +# Check if jsonschema-cli is installed +command -v jsonschema-cli >/dev/null 2>&1 || { + echo >&2 "jsonschema-cli is required but not found. You can install it using 'pip install jsonschema-cli'. Aborting." + exit 1 +} + +# Set default values +ref="main" +local_schema_location="" +datapackage_file="datapackage.json" + +# Parse command-line arguments +while getopts ":r:l:f:" opt; do + case $opt in + r) + ref=$OPTARG + ;; + l) + local_schema_location=$OPTARG + ;; + f) + datapackage_file=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +echo "Validating data package file in $dataset_location" + +# Set the temporary directory path +temp_dir=$(mktemp -d) + +# Set the schema file path based on the option chosen +schema_file="" +if [ "$local_schema_location" != "" ]; then + schema_file="$local_schema_location/tides-data-package.json" +else + # Download the schema file to the temporary directory + schema_url="https://raw.githubusercontent.com/TIDES-transit/TIDES/$ref/spec/tides-data-package.json" + schema_file="$temp_dir/data-package.json" + + if curl -s --head "$schema_url/tides-data-package.json" >/dev/null; then + echo "Schema file not found on GitHub for the specified TIDES version: $tides_version" + exit 1 + fi +curl -o "$schema_file" "$schema_url" +fi + +# Validate datapackage against the downloaded schema +jsonschema-cli validate "$schema_file" "$datapackage_file" + +# Clean up the temporary directory +rm -rf "$temp_dir" diff --git a/docs/datapackage.md b/docs/datapackage.md new file mode 100644 index 00000000..3f04945e --- /dev/null +++ b/docs/datapackage.md @@ -0,0 +1,59 @@ +# Data Package + +TIDES data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json), which is an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/) schema. + +You may create your own `datapackage.json` based on the documentaiton or start with the provided [template](#template), but don't forget to [validate](#validation) it to make sure it is in the correct format! + +## Data Package Format + +{{ frictionless_data_package('spec/tides-data-package.json') }} + +## Tabular Data Resource + +Required and recommended fields for each `tabluar-data-resource` are as follows: + +{{ frictionless_data_package('spec/tides-data-package.json',sub_schema="resources") }} + +## Template + +The canonical `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/TIDES/datapackage.json). + +!!! warning + This version of `tides-data-package` template is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` template if you are viewing the `main` documentation version. + +{{ include_file('samples/template/TIDES/datapackage.json',code_type='json') }} + +## Validation + +There are lots of options for validating your `datapackage.json` file including: + +- [Command Line Interface (CLI) Script](#cli) +- [Various online websites](#point-and-drool) + +### CLI + +You can easily validate your data package file with the script provided in [`/bin/validate-data-package-json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/bin/validate-data-package-json) + +??? tip "installation requirements" + + Make sure you have jsonschema-cli installed. You can install it specifically or with all of the other suggested tools using one of the commands below: + + ```sh + pip install jsonschema-cli + pip install -r requirements.txt + ``` + +```sh title="usage" +validate-data-package-json -f my-datapackage.json +``` + +{{ include_file('bin/validate-data-package-json',code_type='sh') }} + +### Point-and-Drool + +Because a `tides-data-package` is just a json-schema, you can use the myriad of different json-schema validator out there on the web. Use the [canonical `tides-data-package`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) or copy and paste the version from below. + +!!! warning + This version of `tides-data-package` is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` if you are viewing the `main` documentation version. + +{{ include_file('spec/tides-data-package.json',code_type='json') }} diff --git a/docs/samples.md b/docs/samples.md new file mode 100644 index 00000000..7b4125ca --- /dev/null +++ b/docs/samples.md @@ -0,0 +1,9 @@ +# Sample Data + +Sample data can be found in the `/samples` directory, with one directory for each data sample. + +{{ include_file('samples/README.md')}} + +## Data List + +{{ list_samples('samples') }} diff --git a/main.py b/main.py index fada5b1f..38d9037b 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,15 @@ import glob import json +import logging import os import pathlib import re -from typing import Union +from typing import Union, Literal import pandas as pd +log = logging.getLogger("mkdocs") + FIND_REPLACE = { # original relative to /docs : redirect target "": "[Contributing Section](development/#CONTRIBUTING)", "(CODE_OF_CONDUCT.md)": "(development/#CODE_OF_CONDUCT)", @@ -26,9 +29,39 @@ def define_env(env): - macro: a decorator function, to declare a macro. """ + @env.macro + def list_samples(sample_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + Args: + sample_dir (str):directory to search in + Returns: + str: markdown-formatted list + """ + EXCLUDE = ["template"] + fields = ["Sample", "Agency", "Resources", "Vendors"] + + sample_dir = os.path.join(env.project_dir, sample_dir) + samples = [ + d + for d in os.listdir(sample_dir) + if os.path.isdir(os.path.join(sample_dir, d)) and d not in EXCLUDE + ] + + md_table = ( + "| *" + "** | **".join(fields) + "* |\n| " + " ----- |" * len(fields) + "\n" + ) + + for s in samples: + md_table += datapackage_to_row(os.path.join(sample_dir, s, "TIDES"), fields) + return md_table + @env.macro def include_file( - filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None + filename: str, + downshift_h1=True, + start_line: int = 0, + end_line: int = None, + code_type: str = None, ): """ Include a file, optionally indicating start_line and end_line. @@ -42,6 +75,8 @@ def include_file( start_line (Optional): if included, will start including the file from this line (indexed to 0) end_line (Optional): if included, will stop including at this line (indexed to 0) + code_type: if not None, will encapsulate the resulting file in + ```..file contents...``` """ full_filename = os.path.join(env.project_dir, filename) with open(full_filename, "r") as f: @@ -71,26 +106,118 @@ def include_file( _replace = _replace.replace(_filenamebase, "") content = content.replace(_find, _replace) + + if code_type is not None: + content = f"\n```{code_type} title='{filename}'\n{content}\n```" return content + @env.macro + def frictionless_data_package( + data_package_path: str = "**/data-package.json", + sub_schema: str = None, + include: Literal["required", "recommended", "all"] = "recommended", + ) -> str: + """Describes top-level fields of a data package as a markdown table. + + Note: Right now if it finds more than one, it will take the first one. + + Args: + data_package_path (str, optional): location or glob pattern of data package. If glob, + will use the first found match. + sub_schema: if provided, will look for that portion of the schema and use that as + the "top level" + include: specifies the fields to include. Must be one of: + - "required": will only document required fields + - "recommended": will document required and recommended fields + - "all": will document all fields + Defaults to "recommended". + + Returns: a markdown table string + """ + INCLUDE = ["name", "description", "type", "requirement"] + + dp_filename = glob.glob(data_package_path, recursive=True)[0] + + log.info( + f"Documenting spec_file: {dp_filename}.{sub_schema} including {include}" + ) + + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + if sub_schema is not None: + if sub_schema in dp["properties"]: + dp = dp["properties"][sub_schema] + elif sub_schema in dp["$defs"]: + dp = dp["$defs"][sub_schema] + else: + return "Cannot find sub-schema {sub_schema} in data package file {dp_filename}" + if dp["type"] == "array": + dp = dp["items"] + + _field_names = [] + + if include in ["required", "recommended"]: + _field_names += dp.get("required", []) + if include == "recommended": + _field_names += dp.get("recommended", []) + if include == "all": + _field_names = list(dp["properties"].keys()) + + if not _field_names: + return "No fields found to document with parameters." + + _fields = [] + for _f in _field_names: + log.debug(f"Documenting: {_f}") + _row_entry = {k: v for k, v in dp["properties"][_f].items() if k in INCLUDE} + if _f in dp.get("required", []): + _row_entry["requirement"] = "required" + elif _f in dp.get("recommended", []): + _row_entry["requirement"] = "recommended" + else: + _row_entry["requirement"] = " - " + + _row_entry["name"] = f"`{_f}`" + if dp["properties"][_f].get("enum"): + _row_entry["description"] += "
**Must** be one of:
  • `" + _row_entry["description"] += "`
  • `".join( + dp["properties"][_f]["enum"] + ) + _row_entry["description"] += "`
" + elif dp["properties"][_f].get("const"): + _row_entry[ + "description" + ] += f"
**Must** be: `{dp['properties'][_f]['const']}`" + elif dp["properties"][_f].get("examples"): + _ex = dp["properties"][_f]["examples"][0].replace("\n", "
") + _row_entry["description"] += f"
**Example**:
`{_ex}`" + + _fields.append(_row_entry) + + dp_df = pd.DataFrame(_fields, columns=INCLUDE) + dp_md = dp_df.to_markdown(index=False) + + return dp_md + @env.macro def frictionless_spec( spec_path: str = "**/*.spec.json", ) -> str: """Translate the frictionless .spec file to a markdown table. - Note: Right now if it finds more than one, it will take the first one. + Note: Right now if it finds more than one, it will take the first one. If glob, + will use the first found match. Args: - spec_path (str, optional): base path of repo. Defaults to two directories - up from this file. + spec_path (str, optional): location or glob pattern of spec Returns: a markdown table string """ spec_file = glob.glob(spec_path, recursive=True)[0] - print("Documenting spec_file: ", spec_file) + log.info(f"Documenting spec_file: {spec_file}") # Generate a table for overall file requirements spec_df = read_config(spec_file) @@ -117,7 +244,7 @@ def frictionless_schemas( # Create markdown with a table for each schema file schema_files = glob.glob(schema_path, recursive=True) - print("Documenting schemas from: {}".format(schema_files)) + log.info("Documenting schemas from: {schema_files}") file_schema_markdown = [ _document_frictionless_schema(_s) for _s in schema_files @@ -136,6 +263,49 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] +def datapackage_to_row(dir: str, fields: list) -> str: + """Reads datapackage and translates key metadata to a row in a markdown table. + + Right now, only works for fields: "Sample", "Agency", "Resources", "Vendors" + + Args: + dir (str): fully qualified directory for sample data + fields (list): list of fields to return + + Returns: + str: row in a markdown table with name, agency, resources, vendors + """ + dp_filename = os.path.join(dir, "datapackage.json") + if not os.path.isfile(dp_filename): + raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + _vendors = [ + s.get("vendor", None) for r in dp["resources"] for s in r.get("sources", []) + ] + _vendors = list(set(_vendors) - set([None])) + + _md_cells = { + "Sample": _to_sample_readme_link(dp["name"], dir), + "Agency": dp["agency"], + "Resources": ( + "
  • `" + + "`
  • `".join([r["name"] for r in dp["resources"]]) + + "`
" + ), + "Vendors": "
  • `" + "`
  • `".join(_vendors) + "`
", + } + + md_row = "| " + " | ".join([_md_cells[f] for f in fields]) + " |\n" + + return md_row + + +def _to_sample_readme_link(sample_name, folder_dir): + return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" + + def _document_frictionless_schema(schema_filename: str) -> dict: """Documents a single frictionless schema. @@ -149,7 +319,7 @@ def _document_frictionless_schema(schema_filename: str) -> dict: "schema_md": markdown table documenting fields """ - print(f"Documenting schema from file: {schema_filename}") + log.info(f"Documenting schema from file: {schema_filename}") spec_name = schema_filename.split("/")[-1].split(".")[0] spec_title = " ".join([x.capitalize() for x in spec_name.split("_")]) schema = read_schema(schema_filename) diff --git a/mkdocs.yml b/mkdocs.yml index aae3d55c..a818df60 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,6 +6,7 @@ edit_uri: edit/main/docs theme: name: material features: + - content.code.copy - navigation.tabs - navigation.expand - toc.integrate @@ -34,8 +35,14 @@ plugins: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' + - redirects: + redirect_maps: #original relative to /docs : redirect target + '../CODE_OF_CONDUCT.md': 'CODE_OF_CONDUCT.md' + '../CONTRIBUTING.md': 'CONTRIBUTING.md' + '../contributors.md': 'contributors.md' - search + watch: - CONTRIBUTING.md - CODE_OF_CONDUCT.md @@ -74,4 +81,6 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md + - Data Package: datapackage.md + - Sample Data: samples.md - Development: development.md diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..e529f357 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,49 @@ +# Samples Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +```sh +unique-example-name + \TIDES # Required. Data to be validated against the TIDES specification + \datapackage.json # Required. [`tides-data-package`](#tides-data-package) metadata + \raw # Optional. Data which the agency uses to create TIDES data + \scripts # Optional. Scripts used to transform raw --> TIDES +``` + +## Adding Examples + +We encourage the addition of examples, but please follow the following guidelines: + +1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. All individual files should be well under 50 MB. +2. *Include Metadata* as specified in [tides-data-package](#data-package). +3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. + +## Data Package + +TIDES sample data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). + +See: + +- [Full documentation on the `tides-data-package`]({{ site_url }}/datapackage) +- [Example `tides-data-package` file](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template) + +## Data validation + +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: + +```bash +pip install frictionless +frictionless validate --schema-sync path/to/your/datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json +``` + +### Continuous Data Validation + +Sample data in the `\TIDES` subdirectories of each sample is validated upon a push action to the main repository. diff --git a/samples/template/README.md b/samples/template/README.md new file mode 100644 index 00000000..1cdaf7f0 --- /dev/null +++ b/samples/template/README.md @@ -0,0 +1,16 @@ +# Example TIDES Data Package + +Template directory for example scaffolding and helper scripts. + +## Scripts for Generating Template Data + +`scripts\create_template.py` has some template code which can help with the following + +- `write_schema_examples()`: will generate blank csvs according to the TIDES schema +- `write_datapackage()` will generate a datapackage.json based on the TIDES schemas and a set of defaults specified in th script. + +To run both (note this replaces the existing files in the directory) + +```bash +python samples/template/scripts/create_template.py +``` diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json new file mode 100644 index 00000000..0609c7de --- /dev/null +++ b/samples/template/TIDES/datapackage.json @@ -0,0 +1,171 @@ +{ + "name": "template", + "title": "Template TIDES Data Package Example", + "agency": "Transit Agency Name", + "ntd_id": "1234-56", + "profile": "tabular-data-package", + "licenses": [ + { + "name": "Apache-2.0" + } + ], + "contributors": [ + { + "title": "My Name", + "email": "me@myself.com", + "github": "myhandle" + } + ], + "maintainers": [ + { + "title": "Another Name", + "email": "another@myself.com", + "github": "myhandle" + } + ], + "resources": [ + { + "name": "devices", + "path": "devices.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_locations", + "path": "vehicle_locations.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "fare_transactions", + "path": "fare_transactions.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "train_cars", + "path": "train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "operators", + "path": "operators.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "stop_visits", + "path": "stop_visits.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_train_cars", + "path": "vehicle_train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicles", + "path": "vehicles.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "trips_performed", + "path": "trips_performed.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "station_activities", + "path": "station_activities.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "passenger_events", + "path": "passenger_events.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + } + ] +} diff --git a/samples/template/TIDES/devices.csv b/samples/template/TIDES/devices.csv new file mode 100644 index 00000000..6540cce5 --- /dev/null +++ b/samples/template/TIDES/devices.csv @@ -0,0 +1 @@ +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location diff --git a/samples/template/TIDES/fare_transactions.csv b/samples/template/TIDES/fare_transactions.csv new file mode 100644 index 00000000..3f65e125 --- /dev/null +++ b/samples/template/TIDES/fare_transactions.csv @@ -0,0 +1 @@ +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance diff --git a/samples/template/TIDES/operators.csv b/samples/template/TIDES/operators.csv new file mode 100644 index 00000000..713abf28 --- /dev/null +++ b/samples/template/TIDES/operators.csv @@ -0,0 +1 @@ +operator_id diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv new file mode 100644 index 00000000..34932008 --- /dev/null +++ b/samples/template/TIDES/passenger_events.csv @@ -0,0 +1 @@ +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id diff --git a/samples/template/TIDES/station_activities.csv b/samples/template/TIDES/station_activities.csv new file mode 100644 index 00000000..cbbc9d94 --- /dev/null +++ b/samples/template/TIDES/station_activities.csv @@ -0,0 +1 @@ +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv new file mode 100644 index 00000000..2801b88c --- /dev/null +++ b/samples/template/TIDES/stop_visits.csv @@ -0,0 +1 @@ +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship diff --git a/samples/template/TIDES/train_cars.csv b/samples/template/TIDES/train_cars.csv new file mode 100644 index 00000000..3b0e604b --- /dev/null +++ b/samples/template/TIDES/train_cars.csv @@ -0,0 +1 @@ +train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type diff --git a/samples/template/TIDES/trips_performed.csv b/samples/template/TIDES/trips_performed.csv new file mode 100644 index 00000000..ba27e909 --- /dev/null +++ b/samples/template/TIDES/trips_performed.csv @@ -0,0 +1 @@ +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship diff --git a/samples/template/TIDES/vehicle_locations.csv b/samples/template/TIDES/vehicle_locations.csv new file mode 100644 index 00000000..5dff84a7 --- /dev/null +++ b/samples/template/TIDES/vehicle_locations.csv @@ -0,0 +1 @@ +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship diff --git a/samples/template/TIDES/vehicle_train_cars.csv b/samples/template/TIDES/vehicle_train_cars.csv new file mode 100644 index 00000000..9044359c --- /dev/null +++ b/samples/template/TIDES/vehicle_train_cars.csv @@ -0,0 +1 @@ +vehicle_id,train_car_id,order,operator_id diff --git a/samples/template/TIDES/vehicles.csv b/samples/template/TIDES/vehicles.csv new file mode 100644 index 00000000..67ef59e8 --- /dev/null +++ b/samples/template/TIDES/vehicles.csv @@ -0,0 +1 @@ +vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing diff --git a/samples/template/scripts/create_template_files.py b/samples/template/scripts/create_template_files.py new file mode 100644 index 00000000..a9122f36 --- /dev/null +++ b/samples/template/scripts/create_template_files.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +""" +Generates a set of template files using structure and defaults of **schema.json + +--output-path Saves it in the specified output directory (/path/to/output). +If the argument is not provided, it will default to the same directory as the script. + +--schema can specify using a specific schema instead of the default of all +""" + +import argparse +import glob +import logging +import os +import pathlib + +from frictionless import Schema + +USAGE = """ +python samples/template/scripts/create_template_files.py --output-dir /path/to/output +""" + + +SAMPLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_REPO_DIR = os.path.dirname(os.path.dirname(SAMPLE_DIR)) +SCHEMAS = glob.glob( + os.path.join(os.path.join(BASE_REPO_DIR, "spec"), "**/*.schema.json"), + recursive=True, +) +DEFAULT_OUT_PATH = "../TIDES" + + +def write_template_for_schema( + schema_filename: str, + out_dir: str, +) -> None: + """Creates blank csvs which comply with a schema. + + Args: + schema_filename (str): Filename with the Frictionless data schema + out_dir (str): Where the csv will be written + """ + _schema = Schema(schema_filename) + fields = [s.name for s in _schema.fields] + _schema_name = pathlib.Path(schema_filename).stem.split(".")[0] + out_filename = os.path.join(out_dir, _schema_name + ".csv") + with open(out_filename, "w") as outfile: + logging.info(f"Writing: {out_filename}") + outfile.write(",".join(fields)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser(description="Generate a template csvs.") + parser.add_argument( + "--schema", + default=SCHEMAS, + help="List of paths to frictionless schemas to generate. Default: all in /spec dir.", + ) + parser.add_argument( + "--output-path", + default=DEFAULT_OUT_PATH, + help="Directory to save the generated files. Default: same directory as script.", + ) + args = parser.parse_args() + + for s in args.schema: + write_template_for_schema(s, args.output_path) diff --git a/samples/template/scripts/requirements.txt b/samples/template/scripts/requirements.txt new file mode 100644 index 00000000..140b95f1 --- /dev/null +++ b/samples/template/scripts/requirements.txt @@ -0,0 +1 @@ +frictionless diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json index 8d055dd5..40078576 100644 --- a/spec/tides-data-package.json +++ b/spec/tides-data-package.json @@ -1,688 +1,708 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TIDES Data Package", - "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", - "type": "object", - "required": [ - "profile", - "resources", - "title" - ], - "properties": { - "profile": { - "enum": [ - "tides-data-package" - ], - "propertyOrder": 10, - "title": "Profile", - "description": "The profile of this descriptor.", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string", - "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" - ] - }, - "title": { - "propertyOrder": 20, - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "name": { - "propertyOrder": 30, - "title": "Name", - "description": "Short sluggable (https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "{\n \"name\": \"tides-data-package\"\n}\n" - ] - }, - "description": { - "propertyOrder": 40, - "format": "textarea", - "title": "Description", - "description": "Short description of TIDES data package.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "agency": { - "propertyOrder": 50, - "format": "textarea", - "title": "Agency", - "description": "Transit agency name.", - "type": "string", - "examples": [ - "{\n \"agency\": \"Coos County Area Transit\"\n}\n" - ] - }, - "ntd_id": { - "propertyOrder": 60, - "title": "NTD ID", - "description": "ID for the National Transit Database.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "Background on NTD available at https://www.transit.dot.gov/ntd", - "examples": [ - "{\n \"ntd_id\": \"0R02-00307\"\n}\n" - ] - }, - "contributors": { - "propertyOrder": 70, - "title": "Contributors", - "description": "Array of data contributors `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", - "type": "array", - "minItems": 1, - "items": { - "title": "Contributor", - "description": "A contributor to this descriptor.", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "{\n \"title\": \"Frank Julian Sprague\"\n}\n" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this contributor.", - "type": "string" - }, - "role": { - "title": "Role", - "description": "The contributor's role in the project", - "type": "string", - "default": "contributor" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TIDES Data Package", + "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", + "type": "object", + "required": [ + "title", + "profile", + "resources" + + ], + "recommended":[ + "name", + "description", + "agency", + "ntd_id", + "licenses", + "contributors", + "maintainers" + ], + "properties": { + "profile": { + "const": "tides-data-package", + "propertyOrder": 10, + "title": "Profile", + "description": "The json-schema profile used to validate this datapackage descriptor.", + "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", + "type": "string" + }, + "title": { + "propertyOrder": 20, + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "Make Believe Trolley 1/1/2001-2/1/2001" + ] + }, + "name": { + "propertyOrder": 30, + "title": "Name", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "trolley_2001_1" + ] + }, + "description": { + "propertyOrder": 40, + "format": "textarea", + "title": "Description", + "description": "Short description of TIDES data package.", + "type": "string", + "examples": [ + "Raw and processed data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" + ] + }, + "agency": { + "propertyOrder": 50, + "format": "textarea", + "title": "Agency", + "description": "Transit agency name.", + "type": "string", + "examples": [ + "Make Believe Neighborhood Trolley" + ] + }, + "ntd_id": { + "propertyOrder": 60, + "title": "NTD ID", + "description": "ID for the National Transit Database.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "Background on NTD available at https://www.transit.dot.gov/ntd", + "examples": [ + "0R02-00307" + ] + }, + "contributors": { + "propertyOrder": 70, + "title": "Contributors", + "description": "Array of data contributors.", + "type": "array", + "minItems": 1, + "items": { + "title": "Contributor", + "description": "A contributor to this descriptor.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "Daniel Tiger" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "daniel@makebelievetrolley.gov" + ] }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this contributor.", + "type": "string", + "example": "Make Believe Neighborhood Trolley" + }, + "role": { + "title": "Role", + "description": "The contributor's role in the project", + "type": "string", + "default": "contributor" + } }, - "examples": [ - "{\n \"contributors\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"contributors\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" - ] + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." }, - "maintainers": { - "propertyOrder": 80, - "title": "Maintainers", - "description": "Array of data maintainers `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", - "type": "array", - "minItems": 1, - "items": { - "title": "Maintainer", - "description": "A maintainer of this descriptor.", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "{\n \"name\": \"George Francis Train\"\n}\n" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this maintainer.", - "type": "string" - }, - "role": { - "title": "Role", - "description": "The maintainer's role in the project", - "type": "string", - "default": "maintainer" - } + "examples": [ + "{'name':'Daniel Tiger','email': 'daniel@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'contributor'}" + ] + }, + "maintainers": { + "propertyOrder": 80, + "title": "Maintainers", + "description": "Array of data maintainers", + "type": "array", + "minItems": 1, + "items": { + "title": "Maintainer", + "description": "A maintainer of this descriptor.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "Henrietta Pussycat" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "henrietta@makebelievetrolley.gov" + ] + }, + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this maintainer.", + "type": "string" }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." + "role": { + "title": "Role", + "description": "The maintainer's role in the project", + "type": "string", + "default": "maintainer" + } }, - "examples": [ - "{\n \"maintainers\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"maintainers\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" - ] + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." }, - "licenses": { - "propertyOrder": 90, - "title": "Licenses", - "description": "The license(s) under which this package is published.", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "A license for this descriptor.", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a Unix-style relative file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/gpl-2-0/\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - } + "examples": [ + "{'name':'Henrietta Pussycat','email': 'henrietta@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'maintainer'}" + + ] + }, + "licenses": { + "propertyOrder": 90, + "title": "Licenses", + "description": "The license(s) under which this package is published.", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", + "type": "object", + "anyOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "path" + ] } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" - ] + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] + }, + "path": { + "title": "Path", + "description": "A fully qualified public URL, or a Unix-style relative file path.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "https://opendefinition.org/licenses/odc-by/", + "license.md" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "title": { + "title": "Title", + "description": "A human-readable title of the license.", + "type": "string", + "examples": [ + "Creative Commons Attribution 4.0" + ] + } + } }, - "resources": { - "propertyOrder": 100, - "title": "Tabular Data Resources", - "description": "Array of data files included in your package, formated as a tabular-data-resource", - "type": "array", - "minItems": 1, - "items": { - "title": "Tabular Data Resource", - "description": "A Tabular Data Resource.", - "type": "object", - "required": [ + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" + ] + }, + "resources": { + "propertyOrder": 100, + "title": "Tabular Data Resources", + "description": "Array describing the data files included in your package, formated as a [`tabular-data-resource`](#tabular-data-resource)", + "type": "array", + "minItems": 1, + "items": { + "title": "Tabular Data Resource", + "description": "A Tabular Data Resource.", + "type": "object", + "required": [ + "name", + "path", + "schema", + "profile" + ], + "recommended": [ + "title", + "description", + "sources", + "licenses" + ], + "properties": { + "profile": { + "const": "tabular-data-resource", + "propertyOrder": 10, + "title": "Profile", + "description": "Must be `tabular-data-resource`", + "type": "string", + "examples": [ + "tabular-data-resource" + ] + }, + "name": { + "propertyOrder": 20, + "title": "Name", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string for a data table.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "vehicle_locations_2001_01" + ] + }, + "path": { + "propertyOrder": 30, + "title": "Path", + "description": "A reference to the data for this resource, as a valid URI string.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "event/2001/vehicle_locations_2001_01.csv" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "schema": { + "propertyOrder": 40, + "title": "Table Schema", + "description": "An object describing the TIDES Table Schema for this resource.", + "type": "object", + "required": [ "name", - "path", - "schema" - ], - "properties": { - "profile": { - "enum": [ - "tabular-data-resource" - ], - "propertyOrder": 10, - "title": "Profile", - "description": "Should be `tides-data-package`", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string", - "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" - ] - }, - "name": { - "propertyOrder": 20, - "title": "Name", - "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "{\n \"name\": \"tabular-data-resource\"\n}\n" - ] - }, - "path": { - "propertyOrder": 30, - "title": "Path", - "description": "A reference to the data for this resource, as a valid URI string.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "schema": { - "propertyOrder": 40, - "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with one of the [TIDES](/) table specifications.", - "type": "string", - "required": [ - "name", - "path" - ], - "properties": { - "name": { - "propertyOrder": 10, - "title": "Name", - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "examples": [ - "{\n \"name\": \"fare_transactions\"\n}\n", - "{\n \"name\": \"train_cars\"\n}\n" - ] - }, - "path": { - "propertyOrder": 20, - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/fare_transactions.schema.json\"\n}\n", - "{\n \"path\": \"https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json\"\n}\n" - ] - } - }, - "examples": [ - "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" - ] - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ], - "propertyOrder": 50 - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ], - "propertyOrder": 60, - "format": "textarea" - }, - "homepage": { - "propertyOrder": 70, - "title": "Home Page", - "description": "The home on the web that is related to this data package.", - "type": "string", - "format": "uri", - "examples": [ - "{\n \"homepage\": \"http://example.com/\"\n}\n" - ] - }, - "sources": { - "propertyOrder": 140, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "The raw sources for this resource.", - "type": "array", - "minItems": 0, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" + "path" + ], + "properties": { + "name": { + "propertyOrder": 10, + "title": "Name", + "enum": [ + "devices", + "fare_transactions", + "operators", + "station_activities", + "passenger_events", + "stop_visits", + "train_cars", + "trips_performed", + "vehicle_locations", + "vehicle_train_cars", + "vehicles" ], - "properties": { - "title": { - "title": "Title", - "description": "Description of the data source.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "component": { - "title": "Component", - "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", - "type": "string" - }, - "product": { - "title": "Product", - "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string" - }, - "product_version": { - "title": "Product Version", - "description": "Describe the version of the product was used.", - "type": "string" - }, - "vendor": { - "title": "Vendor", - "description": "What company makes this product?", - "type": "string" - } - } + "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "type": "string", + "examples": [ + "fare_transactions", + "train_cars" + ] + }, + "path": { + "propertyOrder": 20, + "title": "Path", + "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "fare_transactions.schema.json" + ] } }, - "licenses": { - "description": "Should be `[{\"name\": \"Apache-2.0\"}]` to be consistent with this repository", - "propertyOrder": 150, - "options": { - "hidden": true - }, - "title": "Licenses", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "A license for this descriptor.", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/apache-2-0/\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - } - } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" - ] + "examples": [ + "`{'name': 'devices' ,'path': 'https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json'}`" + ] + }, + "title": { + "title": "Title", + "description": "A human-readable title for the data resource.", + "type": "string", + "examples": [ + "Neighborhood Trolley Vehicle Locations: 1/1/2001-2/1/2001" + ], + "propertyOrder": 50 + }, + "description": { + "title": "Description", + "description": "D", + "type": "string", + "examples": [ + "Vehicle location event data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" + ], + "propertyOrder": 60, + "format": "textarea" + }, + "homepage": { + "propertyOrder": 70, + "title": "Home Page", + "description": "The home on the web that is related to this data package.", + "type": "string", + "format": "uri", + "examples": [ + "http://example.com" + ] + }, + "sources": { + "propertyOrder": 140, + "options": { + "hidden": true }, - "dialect": { - "propertyOrder": 50, - "title": "CSV Dialect", - "description": "The CSV dialect descriptor.", - "type": [ - "string", - "object" - ], + "title": "Sources", + "description": "The raw sources for this resource which describe where the data came from.", + "type": "array", + "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], + "items": { + "title": "Source", + "type": "object", "required": [ - "delimiter", - "doubleQuote" + "title" ], "properties": { - "csvddfVersion": { - "title": "CSV Dialect schema version", - "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", - "type": "number", - "default": 1.2, - "examples:": [ - "{\n \"csvddfVersion\": \"1.2\"\n}\n" - ] - }, - "delimiter": { - "title": "Delimiter", - "description": "A character sequence to use as the field separator.", + "title": { + "title": "Title", + "description": "Human-readable title of the data source.", "type": "string", - "default": ",", "examples": [ - "{\n \"delimiter\": \",\"\n}\n", - "{\n \"delimiter\": \";\"\n}\n" + "Trolley CAD/AVL System" ] }, - "doubleQuote": { - "title": "Double Quote", - "description": "Specifies the handling of quotes inside fields.", - "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", - "type": "boolean", - "default": true, - "examples": [ - "{\n \"doubleQuote\": true\n}\n" - ] - }, - "lineTerminator": { - "title": "Line Terminator", - "description": "Specifies the character sequence that must be used to terminate rows.", + "component": { + "title": "Component", + "description": "The technology component used to generate this data (directly or indirectly)", "type": "string", - "default": "\r\n", "examples": [ - "{\n \"lineTerminator\": \"\\r\\n\"\n}\n", - "{\n \"lineTerminator\": \"\\n\"\n}\n" + "CAD/AVL", + "AFC", + "APC" ] }, - "nullSequence": { - "title": "Null Sequence", - "description": "Specifies the null sequence, for example, `\\N`.", + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", "type": "string", "examples": [ - "{\n \"nullSequence\": \"\\N\"\n}\n" + "TrolleyMaster" ] }, - "quoteChar": { - "title": "Quote Character", - "description": "Specifies a one-character string to use as the quoting character.", - "type": "string", - "default": "\"", - "examples": [ - "{\n \"quoteChar\": \"'\"\n}\n" - ] - }, - "escapeChar": { - "title": "Escape Character", - "description": "Specifies a one-character string to use as the escape character.", - "type": "string", - "examples": [ - "{\n \"escapeChar\": \"\\\\\"\n}\n" - ] + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" }, - "skipInitialSpace": { - "title": "Skip Initial Space", - "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", - "type": "boolean", - "default": false, - "examples": [ - "{\n \"skipInitialSpace\": true\n}\n" + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" + } + } + } + }, + "licenses": { + "propertyOrder": 90, + "title": "Licenses", + "description": "The license(s) under which this package is published.", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", + "type": "object", + "anyOf": [ + { + "required": [ + "name" ] }, - "header": { - "title": "Header", - "description": "Specifies if the file includes a header row, always as the first row in the file.", - "type": "boolean", - "default": true, - "examples": [ - "{\n \"header\": true\n}\n" + { + "required": [ + "path" ] + } + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] }, - "commentChar": { - "title": "Comment Character", - "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", + "path": { + "title": "Path", + "description": "A fully qualified public URL, or a Unix-style relative file path.", "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"commentChar\": \"#\"\n}\n" - ] + "https://opendefinition.org/licenses/odc-by/", + "license.md" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." }, - "caseSensitiveHeader": { - "title": "Case Sensitive Header", - "description": "Specifies if the case of headers is meaningful.", - "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", - "type": "boolean", - "default": false, + "title": { + "title": "Title", + "description": "A human-readable title of the license.", + "type": "string", "examples": [ - "{\n \"caseSensitiveHeader\": true\n}\n" + "Creative Commons Attribution 4.0" ] } - }, - "examples": [ - "{\n \"dialect\": {\n \"delimiter\": \";\"\n }\n}\n", - "{\n \"dialect\": {\n \"delimiter\": \"\\t\",\n \"quoteChar\": \"'\",\n \"commentChar\": \"#\"\n }\n}\n" - ] - }, - "format": { - "propertyOrder": 80, - "title": "Format", - "description": "The file format of this resource.", - "context": "`csv`, `xls`, `json` are examples of common formats.", - "type": "string", - "examples": [ - "{\n \"format\": \"xls\"\n}\n" - ] + } }, - "mediatype": { - "propertyOrder": 90, - "title": "Media Type", - "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", - "type": "string", - "pattern": "^(.+)/(.+)$", - "examples": [ - "{\n \"mediatype\": \"text/csv\"\n}\n" - ] + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" + ] + }, + "dialect": { + "propertyOrder": 50, + "title": "CSV Dialect", + "description": "The CSV dialect descriptor.", + "type": [ + "string", + "object" + ], + "required": [ + "delimiter", + "doubleQuote" + ], + "properties": { + "csvddfVersion": { + "title": "CSV Dialect schema version", + "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", + "type": "number", + "default": 1.2, + "examples:": [ + "1.2" + ] + }, + "delimiter": { + "title": "Delimiter", + "description": "A character sequence to use as the field separator.", + "type": "string", + "default": ",", + "examples": [ + "`;`", + "`|`" + ] + }, + "doubleQuote": { + "title": "Double Quote", + "description": "Specifies the handling of quotes inside fields.", + "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", + "type": "boolean", + "default": true + }, + "lineTerminator": { + "title": "Line Terminator", + "description": "Specifies the character sequence that must be used to terminate rows.", + "type": "string", + "default": "\r\n", + "examples": [ + "`\r`", + "`\n`" + ] + }, + "nullSequence": { + "title": "Null Sequence", + "description": "Specifies the null sequence, for example, `\\N`.", + "type": "string", + "examples": [ + "`\\N`" + ] + }, + "quoteChar": { + "title": "Quote Character", + "description": "Specifies a one-character string to use as the quoting character.", + "type": "string", + "default": "\"", + "examples": [ + "`'`" + ] + }, + "escapeChar": { + "title": "Escape Character", + "description": "Specifies a one-character string to use as the escape character.", + "type": "string", + "examples": [ + "`\\`" + ] + }, + "skipInitialSpace": { + "title": "Skip Initial Space", + "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", + "type": "boolean", + "default": false + }, + "header": { + "title": "Header", + "description": "Specifies if the file includes a header row, always as the first row in the file.", + "type": "boolean", + "default": true + }, + "commentChar": { + "title": "Comment Character", + "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", + "type": "string", + "examples": [ + "`#`" + ] + }, + "caseSensitiveHeader": { + "title": "Case Sensitive Header", + "description": "Specifies if the case of headers is meaningful.", + "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", + "type": "boolean", + "default": false + } }, - "encoding": { - "propertyOrder": 100, - "title": "Encoding", - "description": "The file encoding of this resource.", - "type": "string", - "default": "utf-8", - "examples": [ - "{\n \"encoding\": \"utf-8\"\n}\n" - ] + "examples": [ + "{'delimiter': ';' }" + ] + }, + "format": { + "propertyOrder": 80, + "title": "Format", + "description": "The file format of this resource.", + "context": "`csv`, `xls`, `json` are examples of common formats.", + "type": "string", + "examples": [ + "`xls`" + ] + }, + "mediatype": { + "propertyOrder": 90, + "title": "Media Type", + "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", + "type": "string", + "pattern": "^(.+)/(.+)$", + "examples": [ + "`text/csv`" + ] + }, + "encoding": { + "propertyOrder": 100, + "title": "Encoding", + "description": "The file encoding of this resource.", + "type": "string", + "default": "utf-8", + "examples": [ + "`utf-8`" + ] + }, + "bytes": { + "propertyOrder": 110, + "options": { + "hidden": true }, - "bytes": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Bytes", - "description": "The size of this resource in bytes.", - "type": "integer", - "examples": [ - "{\n \"bytes\": 2082\n}\n" - ] + "title": "Bytes", + "description": "The size of this resource in bytes.", + "type": "integer", + "examples": [ + "2082" + ] + }, + "hash": { + "propertyOrder": 120, + "options": { + "hidden": true }, - "hash": { - "propertyOrder": 120, - "options": { - "hidden": true - }, - "title": "Hash", - "type": "string", - "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", - "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", - "examples": [ - "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", - "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" - ] - } + "title": "Hash", + "type": "string", + "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", + "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", + "examples": [ + "d25c9c77f588f5dc32059d2da1136c02", + "SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0" + ] } - }, - "examples": [ - "{\n \"resources\": [\n {\n \"name\": \"my-data\",\n \"data\": [\n \"data.csv\"\n ],\n \"schema\": \"tableschema.json\",\n \"mediatype\": \"text/csv\"\n }\n ]\n}\n" - ] + } + } + }, + "sources": { + "propertyOrder": 140, + "options": { + "hidden": true }, - "sources": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "Array of data sources formatted as a source. Recommended to be documented either here at the top-level or for each individual resource", - "type": "array", - "minItems": 0, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "path": { - "title": "Path", - "description": "A fully qualified URL, or a POSIX file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" - ] - } + "title": "Sources", + "description": "The raw sources for this data package which describe where the data came from. Can be alternatively specified at the individual resource-level.", + "type": "array", + "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], + "items": { + "title": "Source", + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "title": "Title", + "description": "Human-readable title of the data source.", + "type": "string", + "examples": [ + "Trolley CAD/AVL System" + ] + }, + "component": { + "title": "Component", + "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", + "type": "string", + "examples": [ + "CAD/AVL", + "AFC", + "APC" + ] + }, + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", + "type": "string", + "examples": [ + "TrolleyMaster" + ] + }, + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" + }, + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" } - }, - "examples": [ - "{\n \"sources\": [\n {\n \"title\": \"World Bank and OECD\",\n \"path\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" - ] + } } } } +} From 227c97355fd637c4b40035491ec62f56071dccf0 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 14 Jul 2023 15:39:00 -0700 Subject: [PATCH 02/75] update template data --- bin/validate-data-package | 6 +++--- bin/validate-data-package-json | 0 samples/template/TIDES/datapackage.json | 13 ++++++++++++- samples/template/TIDES/devices.csv | 2 +- samples/template/TIDES/fare_transactions.csv | 2 +- samples/template/TIDES/passenger_events.csv | 2 +- samples/template/TIDES/station_activities.csv | 2 +- samples/template/TIDES/stop_visits.csv | 2 +- samples/template/TIDES/trips_performed.csv | 2 +- samples/template/TIDES/vehicle_locations.csv | 2 +- spec/tides-data-package.json | 2 +- spec/{table-schema.json => xtable-schema.json} | 0 12 files changed, 23 insertions(+), 12 deletions(-) mode change 100644 => 100755 bin/validate-data-package mode change 100644 => 100755 bin/validate-data-package-json rename spec/{table-schema.json => xtable-schema.json} (100%) diff --git a/bin/validate-data-package b/bin/validate-data-package old mode 100644 new mode 100755 index 7f95647e..21a82236 --- a/bin/validate-data-package +++ b/bin/validate-data-package @@ -36,7 +36,7 @@ done # Create a temporary data package if using a different schema reference or a local schema tmp_datapackage="" -if [ "$tides_version" != "" ] then +if [ "$tides_version" != "" ]; then tmp_datapackage=$(mktemp) cp "$dataset_location/datapackage.json" "$tmp_datapackage" fi @@ -57,10 +57,10 @@ if [ "$tmp_datapackage" != "" ]; then fi # Validate the data package JSON against the TIDES schema -./validate-data-package-json.sh -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" +./validate-data-package-json -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" # Validate the Frictionless Data Package using the Frictionless CLI -frictionless validate "$dataset_location" +frictionless validate --schema-sync "$dataset_location" # Remove the temporary data package file, if applicable if [ "$tmp_datapackage" != "" ]; then diff --git a/bin/validate-data-package-json b/bin/validate-data-package-json old mode 100644 new mode 100755 diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index 0609c7de..a022f0c7 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -3,7 +3,7 @@ "title": "Template TIDES Data Package Example", "agency": "Transit Agency Name", "ntd_id": "1234-56", - "profile": "tabular-data-package", + "profile": "../../spec/tides-data-package.json", "licenses": [ { "name": "Apache-2.0" @@ -26,6 +26,7 @@ "resources": [ { "name": "devices", + "profile": "tabular-data-resource", "path": "devices.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", "sources": [ @@ -39,6 +40,7 @@ }, { "name": "vehicle_locations", + "profile": "tabular-data-resource", "path": "vehicle_locations.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", "sources": [ @@ -52,6 +54,7 @@ }, { "name": "fare_transactions", + "profile": "tabular-data-resource", "path": "fare_transactions.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ @@ -65,6 +68,7 @@ }, { "name": "train_cars", + "profile": "tabular-data-resource", "path": "train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", "sources": [ @@ -78,6 +82,7 @@ }, { "name": "operators", + "profile": "tabular-data-resource", "path": "operators.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", "sources": [ @@ -91,6 +96,7 @@ }, { "name": "stop_visits", + "profile": "tabular-data-resource", "path": "stop_visits.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", "sources": [ @@ -104,6 +110,7 @@ }, { "name": "vehicle_train_cars", + "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", "sources": [ @@ -117,6 +124,7 @@ }, { "name": "vehicles", + "profile": "tabular-data-resource", "path": "vehicles.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", "sources": [ @@ -130,6 +138,7 @@ }, { "name": "trips_performed", + "profile": "tabular-data-resource", "path": "trips_performed.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", "sources": [ @@ -143,6 +152,7 @@ }, { "name": "station_activities", + "profile": "tabular-data-resource", "path": "station_activities.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", "sources": [ @@ -156,6 +166,7 @@ }, { "name": "passenger_events", + "profile": "tabular-data-resource", "path": "passenger_events.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", "sources": [ diff --git a/samples/template/TIDES/devices.csv b/samples/template/TIDES/devices.csv index 6540cce5..f1096555 100644 --- a/samples/template/TIDES/devices.csv +++ b/samples/template/TIDES/devices.csv @@ -1 +1 @@ -device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,device_location diff --git a/samples/template/TIDES/fare_transactions.csv b/samples/template/TIDES/fare_transactions.csv index 3f65e125..ef3b7eea 100644 --- a/samples/template/TIDES/fare_transactions.csv +++ b/samples/template/TIDES/fare_transactions.csv @@ -1 +1 @@ -transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance,service_date,event_timestamp diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv index 34932008..d4ece193 100644 --- a/samples/template/TIDES/passenger_events.csv +++ b/samples/template/TIDES/passenger_events.csv @@ -1 +1 @@ -passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp diff --git a/samples/template/TIDES/station_activities.csv b/samples/template/TIDES/station_activities.csv index cbbc9d94..3928c0fb 100644 --- a/samples/template/TIDES/station_activities.csv +++ b/samples/template/TIDES/station_activities.csv @@ -1 +1 @@ -date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits,service_date diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv index 2801b88c..ecc6c3b7 100644 --- a/samples/template/TIDES/stop_visits.csv +++ b/samples/template/TIDES/stop_visits.csv @@ -1 +1 @@ -date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date diff --git a/samples/template/TIDES/trips_performed.csv b/samples/template/TIDES/trips_performed.csv index ba27e909..94ce7bcd 100644 --- a/samples/template/TIDES/trips_performed.csv +++ b/samples/template/TIDES/trips_performed.csv @@ -1 +1 @@ -date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship,service_date diff --git a/samples/template/TIDES/vehicle_locations.csv b/samples/template/TIDES/vehicle_locations.csv index 5dff84a7..f7936518 100644 --- a/samples/template/TIDES/vehicle_locations.csv +++ b/samples/template/TIDES/vehicle_locations.csv @@ -1 +1 @@ -location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship,event_timestamp diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json index 40078576..1bd00e35 100644 --- a/spec/tides-data-package.json +++ b/spec/tides-data-package.json @@ -258,7 +258,7 @@ ], "properties": { "profile": { - "const": "tabular-data-resource", + "default": "tabular-data-resource", "propertyOrder": 10, "title": "Profile", "description": "Must be `tabular-data-resource`", diff --git a/spec/table-schema.json b/spec/xtable-schema.json similarity index 100% rename from spec/table-schema.json rename to spec/xtable-schema.json From 343060f031c0f246d9366c99d9601b9105b45d89 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 21 Jul 2023 17:46:53 -0700 Subject: [PATCH 03/75] Working tests and scripts - rename tides-data-package.json --> tides-datapackage-profile.json to be consistent with how other frictionless profiles are named and reduce confusion with an actual data package. - Add 4 shell scripts to /bin: (1) utilities to check presence of packages, files etc. and provide good help messages (2) update a datapackage file temporarily to point to another spec location (3) validate a datapackage file to the tides datapackage profile (4) validate a datapackage and its contents - Update contributing.md documentation - Removes table-schema.json b/c I don't thik we needed it - Updates tests/ files to have three primary scripts to run: (1) test_local_spec which valdiates the local spec (2) test_samples_to_canonical which validates the local samples to the canonical spec and (3) test_samples_to_local which validates local samples to local spec. - Updates table schemas to be a string location to be consistent with most other profiles rather than an object. --- .github/workflows/validate_package_schema.yml | 2 +- .github/workflows/validate_table_schemas.yml | 2 +- CONTRIBUTING.md | 11 +- bin/replace-spec-in-datapackage | 98 + bin/utils | 65 + bin/validate-data-package | 68 - bin/validate-data-package-json | 64 - bin/validate-datapackage | 125 ++ bin/validate-datapackage-to-profile | 138 ++ contributors/index.html | 15 + docs/datapackage.md | 12 +- samples/README.md | 2 +- samples/template/TIDES/datapackage.json | 4 +- ...ge.json => tides-datapackage-profile.json} | 78 +- spec/xtable-schema.json | 1676 ----------------- tests/test_all | 49 - tests/test_local_spec | 61 + tests/test_samples_to_canonical | 25 + tests/test_samples_to_local | 25 + tests/validate_profile | 15 +- tests/validate_samples | 98 +- 21 files changed, 705 insertions(+), 1928 deletions(-) create mode 100755 bin/replace-spec-in-datapackage create mode 100755 bin/utils delete mode 100755 bin/validate-data-package delete mode 100755 bin/validate-data-package-json create mode 100755 bin/validate-datapackage create mode 100755 bin/validate-datapackage-to-profile create mode 100644 contributors/index.html rename spec/{tides-data-package.json => tides-datapackage-profile.json} (91%) delete mode 100644 spec/xtable-schema.json delete mode 100755 tests/test_all create mode 100755 tests/test_local_spec create mode 100755 tests/test_samples_to_canonical create mode 100755 tests/test_samples_to_local diff --git a/.github/workflows/validate_package_schema.yml b/.github/workflows/validate_package_schema.yml index c4d44051..fcab3cb3 100644 --- a/.github/workflows/validate_package_schema.yml +++ b/.github/workflows/validate_package_schema.yml @@ -17,7 +17,7 @@ jobs: uses: TIDES-transit/json-schema-validator@master with: token: ${{ secrets.GITHUB_TOKEN }} - json_schema: spec/tides-data-package.json + json_schema: spec/tides-datapackage-profile.json json_path_pattern: ^spec/tides\.spec\.json$ send_comment: false clear_comments: false diff --git a/.github/workflows/validate_table_schemas.yml b/.github/workflows/validate_table_schemas.yml index f64da12b..d65e1942 100644 --- a/.github/workflows/validate_table_schemas.yml +++ b/.github/workflows/validate_table_schemas.yml @@ -21,4 +21,4 @@ jobs: json_schema: spec/table-schema.json json_path_pattern: ^spec/.*\.schema\.json$ send_comment: false - clear_comments: false \ No newline at end of file + clear_comments: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 785c6026..7047c7d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,9 +34,10 @@ By making any contribution to the projects, contributors self-certify to the [Co 1. [Create a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) to work on a new issue (or checkout an existing one where the issue is being worked on). 2. Make your changes. -3. Run `/tests/test_all` script to check and fix formatting, validate schemas, and build documentation locally to preview -4. [Commit](#commits) your work in `git` -5. `push` your changes to Github and submit a [`pull request`](#pull-requests) +3. Run `tests/test_local_spec` script to check and fix formatting, validate profile and schemas with frictionless and with each other, and confirm that documentation can be built locally. +4. Run `tests/test_samples_to_local` script to check if samples conform to any changes to the spec. +5. [Commit](#commits) your work in `git` +6. `push` your changes to Github and submit a [`pull request`](#pull-requests) ### Issues @@ -105,7 +106,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de | **Name** | **What it does** | | -------- | ----------------- | | GitHub Actions | Runs following workflow on each push to the TIDES github repository: /.github/workflows/docs.yml | - | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc)
For new branches with documentation, adds an entry in `versions.json` | + | mike | runs mkdocs and puts output in a folder in gh_pages branch which corresponds to the name of the branch (i.e. main, develop, pr-163, etc)
For new branches with documentation, adds an entry in `versions.json` | | `mkdocs` | Package which generates documentation from markdown and code | ??? info "Overview of Documentation Building Process" @@ -116,7 +117,7 @@ When a change is pushed to the TIDES specification repository, Github Actions de subgraph mkdocs["mkdocs: run on execution of mike"] md_mike["mike"] -->|runs for current branch| md_mkdocs["mkdocs"] md_mkdocs.yml["mkdocs.yml"] -->|specifieds parameters| md_mkdocs["mkdocs"] - md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"] + md_mkdocs_macros["mkdocs-macros"] -->|"plugin for"| md_mkdocs["mkdocs"] main.py[/"main.py"/] -->|defines macros in code available for| md_mkdocs_macros["mkdocs-macros"] end diff --git a/bin/replace-spec-in-datapackage b/bin/replace-spec-in-datapackage new file mode 100755 index 00000000..889800ed --- /dev/null +++ b/bin/replace-spec-in-datapackage @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +description="Create a temporary data package if a spec_path_prefix is provided." + +usage=" +Usage: replace-spec-in-datapackage [spec_path_prefix] [output_file] + +Arguments: + The path to the dataset directory containing the 'datapackage.json' file. + The path or URL to the spec to be referenced in the updated data package. + [output_file] (Optional) The path to save the temporary data package. If not provided, the temporary data package will be saved as 'datapackage.tmp.json' in the dataset directory. + +" + +example_usage=" +Example Usage: +bin/replace-spec-in-datapackage samples/template/TIDES spec samples/template/TIDES/datapackage.tmp.json +" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# SET DEFAULTS # +################################################################################ +DEFAULT_TMP_DATAPACKAGE="datapackage.tmp.json" +PROFILE_FILE="tides-datapackage-profile.json" + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Check Requirements # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +check_jq + +################################################################################ +# Process the input options. # +################################################################################ +dataset_path=$1 +spec_path_prefix=$2 +output_file=${3:-"$dataset_path/$DEFAULT_TMP_DATAPACKAGE"} + +echo "Parameters: + dataset_path: $dataset_path + spec_path_prefix: $spec_path_prefix + output_file: $output_file +" + +# Check if required arguments are missing +if [ -z "$dataset_path" ]; then + echo "Error: Missing dataset_path argument." >&2 + display_help + exit 1 +fi + +# Check if required arguments are missing +if [ -z "spec_path_prefix" ]; then + echo "Error: Missing spec_path_prefix argument." >&2 + display_help + exit 1 +fi + +datapackage_file="$dataset_path/datapackage.json" + +check_valid_path "$datapackage_file" + +profile_file="$spec_path_prefix/$PROFILE_FILE" +check_valid_path "$profile_file" + +################################################################################ +# Create updated datapackage # +################################################################################ +cp "$datapackage_file" "$output_file" +jq --arg spec_path_prefix "$spec_path_prefix" --arg profile_file "$profile_file" ' + .resources |= map(.schema |= ($spec_path_prefix + "/\(. | split("/") | last)")) + | .profile = ($profile_file) +' "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file" + +echo "$output_file" \ No newline at end of file diff --git a/bin/utils b/bin/utils new file mode 100755 index 00000000..ebc4dc7a --- /dev/null +++ b/bin/utils @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Check if jsonschema-cli is installed +check_jsonschema-cli() { + if ! command -v jsonschema-cli >/dev/null 2>&1; then + echo >&2 "\033[31m!!! jsonschema-cli is required but not found.\033[0m + + You can install it using 'pip install jsonschema-cli'. Aborting." + exit 1 + fi +} + +# Function to check jq and provide installation instructions if not found +check_jq() { + if ! command -v jq >/dev/null 2>&1; then + echo >&2 "\033[31m!!! jq is required but not found.\033[0m" + + # Determine the operating system + os_type=$(uname -s) + + case "$os_type" in + Linux*) + echo >&2 "Please install jq using your package manager." + echo >&2 "For example, on Debian/Ubuntu-based systems, you can use:" + echo >&2 "sudo apt-get install jq" + ;; + Darwin*) + echo >&2 "Please install jq using Homebrew." + echo >&2 "If you don't have Homebrew installed, you can install it from https://brew.sh/." + echo >&2 "Once Homebrew is installed, run the following command:" + echo >&2 "brew install jq" + ;; + CYGWIN*|MINGW32*|MSYS*|MINGW*) + echo >&2 "Please download the jq binary for Windows from https://stedolan.github.io/jq/download/." + echo >&2 "Extract the downloaded ZIP file and add the 'jq.exe' binary to your PATH..." + echo >&2 "Or run the following command: " + echo >&2 "curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe" + ;; + *) + echo >&2 "Unknown operating system. Please install jq from https://stedolan.github.io/jq/download/." + ;; + esac + + exit 1 + fi +} + +# Function to check if this is a valid spec path +check_valid_path() { + local path=$1 + + if [ -n "$path" ]; then + if [ ! -e "$path" ] && [[ ! "$path" =~ ^http ]]; then + echo "\033[31mError: $path is an invalid path. It must be a valid file path, directory path, or a URL starting with 'http'.\033[0m" >&2 + display_help + exit 1 + elif [[ "$path" =~ ^http ]]; then + # Check if the URL is valid + if ! curl --output /dev/null --silent --head --fail "$path"; then + echo "\033[31mError: Invalid URL. The $path URL is not reachable or does not exist.\033[0m" >&2 + exit 1 + fi + fi + fi +} diff --git a/bin/validate-data-package b/bin/validate-data-package deleted file mode 100755 index 21a82236..00000000 --- a/bin/validate-data-package +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# Script: validate_frictionless_package.sh -# Description: Bash script to validate a Frictionless Data Package using the Frictionless CLI. -# Usage: validate_frictionless_package.sh [-v tides_version | -l local_schema_location] [-d dataset_location] -# -v tides_version: Optional. Specify the version of the TIDES specification or 'local' to -# use a local schema. Default is to use the schema specified in the datapackage. -# -l local_schema_location: Optional. Specify the location of the local schema directory. -# Default is '../spec'. Is only used if tides_version = local. -# -d dataset_location: Optional. Specify the location of the TIDES datapackage.json. -# Default is the current directory. - -# Set default values -tides_version="" -local_schema_location="../spec" -dataset_location="." - -# Parse command-line arguments -while getopts ":v:l:d:" opt; do - case $opt in - v) - tides_version=$OPTARG - ;; - l) - local_schema_location=$OPTARG - ;; - d) - dataset_location=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done - -# Create a temporary data package if using a different schema reference or a local schema -tmp_datapackage="" -if [ "$tides_version" != "" ]; then - tmp_datapackage=$(mktemp) - cp "$dataset_location/datapackage.json" "$tmp_datapackage" -fi - -# Set the schema URL based on the option chosen -schema_url="" -if [ "$tides_version" == "local" ]; then - schema_path_prefix="$local_schema_location" -else - schema_path_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$tides_version/spec" -fi - -# Update the 'schema' property in the temporary copy of the datapackage.json file, if applicable -if [ "$tmp_datapackage" != "" ]; then - schema_file=$(echo "$tmp_datapackage" | sed 's/\//\\\//g') - sed -E -i "s/\"schema\": \"[^\/]+\.schema\.json\"/\"schema\": \"$schema_path_prefix\/\${schema_file##*\/}\"/g" "$tmp_datapackage" - dataset_location="$tmp_datapackage" -fi - -# Validate the data package JSON against the TIDES schema -./validate-data-package-json -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" - -# Validate the Frictionless Data Package using the Frictionless CLI -frictionless validate --schema-sync "$dataset_location" - -# Remove the temporary data package file, if applicable -if [ "$tmp_datapackage" != "" ]; then - rm "$tmp_datapackage" -fi diff --git a/bin/validate-data-package-json b/bin/validate-data-package-json deleted file mode 100755 index 504e08ca..00000000 --- a/bin/validate-data-package-json +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# Script to validate a local JSON file against a schema specified in a GitHub repository. -# Usage: validate-data-package-json.sh [-r ref | -l local_schema_location] [-f datapackage_file] -# -r ref: Optional. Specify the ref name of the GitHub repository. Default is 'main'. -# -l local_schema_location: Optional. Specify the location of the local schema directory. -# -f datapackage_file: Optional. Specify the location of the datapackage.json file. Default is 'datapackage.json' in the execution directory. - -# Check if jsonschema-cli is installed -command -v jsonschema-cli >/dev/null 2>&1 || { - echo >&2 "jsonschema-cli is required but not found. You can install it using 'pip install jsonschema-cli'. Aborting." - exit 1 -} - -# Set default values -ref="main" -local_schema_location="" -datapackage_file="datapackage.json" - -# Parse command-line arguments -while getopts ":r:l:f:" opt; do - case $opt in - r) - ref=$OPTARG - ;; - l) - local_schema_location=$OPTARG - ;; - f) - datapackage_file=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - esac -done - -echo "Validating data package file in $dataset_location" - -# Set the temporary directory path -temp_dir=$(mktemp -d) - -# Set the schema file path based on the option chosen -schema_file="" -if [ "$local_schema_location" != "" ]; then - schema_file="$local_schema_location/tides-data-package.json" -else - # Download the schema file to the temporary directory - schema_url="https://raw.githubusercontent.com/TIDES-transit/TIDES/$ref/spec/tides-data-package.json" - schema_file="$temp_dir/data-package.json" - - if curl -s --head "$schema_url/tides-data-package.json" >/dev/null; then - echo "Schema file not found on GitHub for the specified TIDES version: $tides_version" - exit 1 - fi -curl -o "$schema_file" "$schema_url" -fi - -# Validate datapackage against the downloaded schema -jsonschema-cli validate "$schema_file" "$datapackage_file" - -# Clean up the temporary directory -rm -rf "$temp_dir" diff --git a/bin/validate-datapackage b/bin/validate-datapackage new file mode 100755 index 00000000..5d940d6a --- /dev/null +++ b/bin/validate-datapackage @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +description="Bash script to validate a Frictionless Data Package using the Frictionless CLI." + +usage=" +Usage: validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path] + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile. Should not be used with -l option. Example: `-r main` + -l local_spec_path: Optional. Specify the path of the local schema directory. + Default is '../spec'. Is only used if remote_spec_ref = local. + -d dataset_path: Optional. Specify the path of the TIDES datapackage.json. + Default is the current directory. +" + +example_usage="Example Usage: bin/validate-datapackage -l spec -d samples/template/TIDES" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Source utility functions # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +script_dir=$(dirname "$0") + +################################################################################ +# Process the parameters + input options. # +################################################################################ +remote_spec_ref="" +local_spec_path="" +dataset_path="." + +# Parse command-line arguments +while getopts ":r:l:d:" opt; do + case $opt in + r) + remote_spec_ref=$OPTARG + ;; + l) + local_spec_path=$OPTARG + ;; + d) + dataset_path=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +echo "Running validate-datapackage with configs:" +echo " remote_spec_ref: $remote_spec_ref" +echo " local_spec_path: $local_spec_path" +echo " dataset_path: $dataset_path" + +if [ "$local_spec_path" != "" ] && [ "$remote_spec_ref" != "" ]; then + echo "Cannot specify both local profile location (-l) and remote spec reference (-r)" >&2 + exit 1 +fi + +# Convert local_spec_path to an absolute path +if [ -n "$local_spec_path" ]; then + local_spec_path=$(realpath "$local_spec_path") +fi + +################################################################################ +# Specify datapackage and update if necessary # +################################################################################ + +datapackage_file="$dataset_path/datapackage.json" +check_valid_path "$datapackage_file" + +[ -n "$local_spec_path" ] && spec_path_prefix="$local_spec_path" +[ -n "$remote_spec_ref" ] && spec_path_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$remote_spec_ref/spec" + +# Create a temporary data package if using a different spec than specified in datapackage +if [ "$spec_path_prefix" != "" ]; then + tmp_datapackage=$("$script_dir/replace-spec-in-datapackage" "$dataset_path" "$spec_path_prefix" 2>&1 | tee /dev/tty | tail -1) + echo "...using Temporary datapackage at: $tmp_datapackage" + check_valid_path "$tmp_datapackage" + datapackage_file="$tmp_datapackage" +fi + +################################################################################ +# Validate datapackage file against TIDES profile # +################################################################################ + +echo "...validate the data package JSON against the TIDES profile" + +if ! "$script_dir"/validate-datapackage-to-profile -d "$datapackage_file"; then + echo -e "\033[31m!!! ERROR: Data package validation against the TIDES profile failed.\033[0m" +fi + +################################################################################ +# Validate datapackage contents # +################################################################################ +echo "...validate the Frictionless Data Package using the Frictionless CLI" +frictionless validate --schema-sync "$datapackage_file" --trusted + +# Remove the temporary data package file, if applicable +if [ "$tmp_datapackage" != "" ]; then + rm "$tmp_datapackage" +fi diff --git a/bin/validate-datapackage-to-profile b/bin/validate-datapackage-to-profile new file mode 100755 index 00000000..24d92feb --- /dev/null +++ b/bin/validate-datapackage-to-profile @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +description="Script to validate a local JSON file against a profile for tides-data-package specified in +profile field or optionally against a remote or local profile." + +usage=" +Usage: validate-datapackage-to-profile [-r remote_spec_ref | -l local_spec_path] [-f datapackage_file] + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile where the profile is in the sub-path /spec/tides-data-package.json. + Should not be used with -l option. Example: -r main + -l local_spec_path: Optional. Specify the location of the local tides-data-package-json to use. + Should not be used with -r option. Example: -l spec + -d dataset_path: Optional. Specify the path of the datapackage.json file. + Default is datapackage.json. Example: -d samples/template/TIDES/datapackage.json +" + +example_usage="Example Usage: bin/validate-datapackage-to-profile -d samples/template/TIDES/datapackage.tmp.json" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# MAIN # +################################################################################ + +echo "$description" + +################################################################################ +# Source utility functions # +################################################################################ +source "$(dirname "${BASH_SOURCE[0]}")/utils" +script_dir=$(dirname "$0") + +################################################################################ +# Check Requirements # +################################################################################ +check_jsonschema-cli +check_jq + +################################################################################ +# Process the input options. # +################################################################################ +# Set default values +remote_spec_ref="" +local_spec_path="" +dataset_path="datapackage.json" + +# Parse command-line arguments +while getopts ":r:l:d:" opt; do + case $opt in + r) + remote_spec_ref=$OPTARG + ;; + l) + local_spec_path=$OPTARG + ;; + d) + dataset_path=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +echo "validate-datapackage-to-profile configs:" +echo " remote_spec_ref: $remote_profile_ref" +echo " local_spec_path: $local_profile_path" +echo " dataset_path: $dataset_path" + +if [ "$local_spec_path" != "" ] && [ "$remote_spec_path" != "" ]; then + echo "Cannot specify both local spec location (-l) and remote spec reference (-r)" >&2 + exit 1 +fi + +################################################################################ +# Specify datapackage and update if necessary # +################################################################################ + +# Set appropriate datapackage_file vs dataset_path +if [ -f "$dataset_path" ]; then + datapackage_file="$dataset_path" + dataset_path=$(dirname "$dataset_path") +else + datapackage_file="$dataset_path/datapackage.json" +fi +check_valid_path "$datapackage_file" + +################################################################################ +# Find and download if necessary the profile file # +################################################################################ + +[ -n "$local_spec_path" ] && spec_prefix="$local_spec_path" +[ -n "$remote_spec_ref" ] && spec_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$remote_spec_ref/spec" + +if [ "$spec_prefix" != "" ]; then + profile_file="$spec_prefix/tides-datapackage-profile.json" +else + profile_file=$(jq -r '.profile' "$datapackage_file") +fi +echo "...using profile: $profile_file" +check_valid_path "$profile_file" + +if [[ "$profile_file" =~ ^http ]]; then + temp_dir=$(mktemp -d) + profile_file="$temp_dir/$(basename $profile_file)" + echo "...downloading profile to: $profile_file" + curl -L "$profile_file" -o "$profile_file" +fi + +################################################################################ +# Validate Datapackage to Profile # +################################################################################ +echo "...validating $datapackage_file to $profile_file" +jsonschema-cli validate "$profile_file" "$datapackage_file" + +################################################################################ +# Cleanup # +################################################################################ +if [ -d "$temp_dir" ]; then + rm -r "$temp_dir" +fi diff --git a/contributors/index.html b/contributors/index.html new file mode 100644 index 00000000..1ba8159d --- /dev/null +++ b/contributors/index.html @@ -0,0 +1,15 @@ + + + + + + Redirecting... + + + + + + +Redirecting... + + diff --git a/docs/datapackage.md b/docs/datapackage.md index 3f04945e..63025fed 100644 --- a/docs/datapackage.md +++ b/docs/datapackage.md @@ -1,18 +1,18 @@ # Data Package -TIDES data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json), which is an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/) schema. +TIDES data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json), which is an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/) schema. You may create your own `datapackage.json` based on the documentaiton or start with the provided [template](#template), but don't forget to [validate](#validation) it to make sure it is in the correct format! ## Data Package Format -{{ frictionless_data_package('spec/tides-data-package.json') }} +{{ frictionless_data_package('spec/tides-datapackage-profile.json') }} ## Tabular Data Resource Required and recommended fields for each `tabluar-data-resource` are as follows: -{{ frictionless_data_package('spec/tides-data-package.json',sub_schema="resources") }} +{{ frictionless_data_package('spec/tides-datapackage-profile.json',sub_schema="resources") }} ## Template @@ -51,9 +51,9 @@ validate-data-package-json -f my-datapackage.json ### Point-and-Drool -Because a `tides-data-package` is just a json-schema, you can use the myriad of different json-schema validator out there on the web. Use the [canonical `tides-data-package`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) or copy and paste the version from below. +Because a `tides-datapackage-profile` is just a json-schema, you can use the myriad of different json-schema validator out there on the web. Use the [canonical `tides-datapackage-profile`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json) or copy and paste the version from below. !!! warning - This version of `tides-data-package` is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` if you are viewing the `main` documentation version. + This version of `tides-datapackage-profile` is dependent on the version of the documentation you are viewing and only represents the canonical `tides-datapackage-profile` if you are viewing the `main` documentation version. -{{ include_file('spec/tides-data-package.json',code_type='json') }} +{{ include_file('spec/tides-datapackage-profile.json',code_type='json') }} diff --git a/samples/README.md b/samples/README.md index e529f357..09b51056 100644 --- a/samples/README.md +++ b/samples/README.md @@ -20,7 +20,7 @@ We encourage the addition of examples, but please follow the following guideline ## Data Package -TIDES sample data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). +TIDES sample data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json) (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). See: diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index a022f0c7..dd0e5cca 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -3,7 +3,7 @@ "title": "Template TIDES Data Package Example", "agency": "Transit Agency Name", "ntd_id": "1234-56", - "profile": "../../spec/tides-data-package.json", + "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", "licenses": [ { "name": "Apache-2.0" @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", diff --git a/spec/tides-data-package.json b/spec/tides-datapackage-profile.json similarity index 91% rename from spec/tides-data-package.json rename to spec/tides-datapackage-profile.json index 1bd00e35..aa25313b 100644 --- a/spec/tides-data-package.json +++ b/spec/tides-datapackage-profile.json @@ -20,12 +20,14 @@ ], "properties": { "profile": { - "const": "tides-data-package", + "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", "propertyOrder": 10, "title": "Profile", - "description": "The json-schema profile used to validate this datapackage descriptor.", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string" + "description": "The json-schema profile used to validate this datapackage descriptor. Should be `https://raw.githubusercontent.com/TIDES-transit/TIDES//spec/tides-datapackage-profile.json`. Defaults to `https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`", + "format": "uri", + "examples": [ + "`https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`" + ] }, "title": { "propertyOrder": 20, @@ -289,53 +291,33 @@ ], "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." }, + "tides-table": { + "enum": [ + "devices", + "fare_transactions", + "operators", + "station_activities", + "passenger_events", + "stop_visits", + "train_cars", + "trips_performed", + "vehicle_locations", + "vehicle_train_cars", + "vehicles" + ], + "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "type": "string", + "examples": [ + "fare_transactions", + "train_cars" + ] + }, "schema": { - "propertyOrder": 40, "title": "Table Schema", - "description": "An object describing the TIDES Table Schema for this resource.", - "type": "object", - "required": [ - "name", - "path" - ], - "properties": { - "name": { - "propertyOrder": 10, - "title": "Name", - "enum": [ - "devices", - "fare_transactions", - "operators", - "station_activities", - "passenger_events", - "stop_visits", - "train_cars", - "trips_performed", - "vehicle_locations", - "vehicle_train_cars", - "vehicles" - ], - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", - "type": "string", - "examples": [ - "fare_transactions", - "train_cars" - ] - }, - "path": { - "propertyOrder": 20, - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", - "fare_transactions.schema.json" - ] - } - }, + "description": "Location of the table schema for this resource.", + "type": "string", "examples": [ - "`{'name': 'devices' ,'path': 'https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json'}`" + "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json" ] }, "title": { diff --git a/spec/xtable-schema.json b/spec/xtable-schema.json deleted file mode 100644 index a87b49e5..00000000 --- a/spec/xtable-schema.json +++ /dev/null @@ -1,1676 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with the [Table Schema](/tableschema/) specification.", - "type": [ - "string", - "object" - ], - "required": [ - "fields" - ], - "properties": { - "fields": { - "type": "array", - "minItems": 1, - "items": { - "title": "Table Schema Field", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "String Field", - "description": "The field contains strings, that is, sequences of characters.", - "required": [ - "name" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `string`.", - "enum": [ - "string" - ] - }, - "format": { - "description": "The format keyword options for `string` are `default`, `email`, `uri`, `binary`, and `uuid`.", - "context": "The following `format` options are supported:\n * **default**: any valid string.\n * **email**: A valid email address.\n * **uri**: A valid URI.\n * **binary**: A base64 encoded string representing binary data.\n * **uuid**: A string that is a uuid.", - "enum": [ - "default", - "email", - "uri", - "binary", - "uuid" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `string` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "pattern": { - "type": "string", - "description": "A regular expression pattern to test each value of the property against, where a truthy response indicates validity.", - "context": "Regular expressions `SHOULD` conform to the [XML Schema regular expression syntax](http://www.w3.org/TR/xmlschema-2/#regexs)." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"name\",\n \"type\": \"string\"\n}\n", - "{\n \"name\": \"name\",\n \"type\": \"string\",\n \"format\": \"email\"\n}\n", - "{\n \"name\": \"name\",\n \"type\": \"string\",\n \"constraints\": {\n \"minLength\": 3,\n \"maxLength\": 35\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Number Field", - "description": "The field contains numbers of any kind including decimals.", - "context": "The lexical formatting follows that of decimal in [XMLSchema](https://www.w3.org/TR/xmlschema-2/#decimal): a non-empty finite-length sequence of decimal digits separated by a period as a decimal indicator. An optional leading sign is allowed. If the sign is omitted, '+' is assumed. Leading and trailing zeroes are optional. If the fractional part is zero, the period and following zero(es) can be omitted. For example: '-1.23', '12678967.543233', '+100000.00', '210'.\n\nThe following special string values are permitted (case does not need to be respected):\n - NaN: not a number\n - INF: positive infinity\n - -INF: negative infinity\n\nA number `MAY` also have a trailing:\n - exponent: this `MUST` consist of an E followed by an optional + or - sign followed by one or more decimal digits (0-9)\n - percentage: the percentage sign: `%`. In conversion percentages should be divided by 100.\n\nIf both exponent and percentages are present the percentage `MUST` follow the exponent e.g. '53E10%' (equals 5.3).", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `number`.", - "enum": [ - "number" - ] - }, - "format": { - "description": "There are no format keyword options for `number`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "bareNumber": { - "type": "boolean", - "title": "bareNumber", - "description": "a boolean field with a default of `true`. If `true` the physical contents of this field must follow the formatting constraints already set out. If `false` the contents of this field may contain leading and/or trailing non-numeric characters (which implementors MUST therefore strip). The purpose of `bareNumber` is to allow publishers to publish numeric data that contains trailing characters such as percentages e.g. `95%` or leading characters such as currencies e.g. `€95` or `EUR 95`. Note that it is entirely up to implementors what, if anything, they do with stripped text.", - "default": true - }, - "decimalChar": { - "type": "string", - "description": "A string whose value is used to represent a decimal point within the number. The default value is `.`." - }, - "groupChar": { - "type": "string", - "description": "A string whose value is used to group digits within the number. The default value is `null`. A common value is `,` e.g. '100,000'." - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `number` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "number" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "number" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"field-name\",\n \"type\": \"number\"\n}\n", - "{\n \"name\": \"field-name\",\n \"type\": \"number\",\n \"constraints\": {\n \"enum\": [ \"1.00\", \"1.50\", \"2.00\" ]\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Integer Field", - "description": "The field contains integers - that is whole numbers.", - "context": "Integer values are indicated in the standard way for any valid integer.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `integer`.", - "enum": [ - "integer" - ] - }, - "format": { - "description": "There are no format keyword options for `integer`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "bareNumber": { - "type": "boolean", - "title": "bareNumber", - "description": "a boolean field with a default of `true`. If `true` the physical contents of this field must follow the formatting constraints already set out. If `false` the contents of this field may contain leading and/or trailing non-numeric characters (which implementors MUST therefore strip). The purpose of `bareNumber` is to allow publishers to publish numeric data that contains trailing characters such as percentages e.g. `95%` or leading characters such as currencies e.g. `€95` or `EUR 95`. Note that it is entirely up to implementors what, if anything, they do with stripped text.", - "default": true - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `integer` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "integer" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"age\",\n \"type\": \"integer\",\n \"constraints\": {\n \"unique\": true,\n \"minimum\": 100,\n \"maximum\": 9999\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Date Field", - "description": "The field contains temporal date values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `date`.", - "enum": [ - "date" - ] - }, - "format": { - "description": "The format keyword options for `date` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string of YYYY-MM-DD.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `date` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\"\n}\n", - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\",\n \"constraints\": {\n \"minimum\": \"01-01-1900\"\n }\n}\n", - "{\n \"name\": \"date_of_birth\",\n \"type\": \"date\",\n \"format\": \"MM-DD-YYYY\"\n}\n" - ] - }, - { - "type": "object", - "title": "Time Field", - "description": "The field contains temporal time values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `time`.", - "enum": [ - "time" - ] - }, - "format": { - "description": "The format keyword options for `time` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string for time.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `time` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"appointment_start\",\n \"type\": \"time\"\n}\n", - "{\n \"name\": \"appointment_start\",\n \"type\": \"time\",\n \"format\": \"any\"\n}\n" - ] - }, - { - "type": "object", - "title": "Date Time Field", - "description": "The field contains temporal datetime values.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `datetime`.", - "enum": [ - "datetime" - ] - }, - "format": { - "description": "The format keyword options for `datetime` are `default`, `any`, and `{PATTERN}`.", - "context": "The following `format` options are supported:\n * **default**: An ISO8601 format string for datetime.\n * **any**: Any parsable representation of a date. The implementing library can attempt to parse the datetime via a range of strategies.\n * **{PATTERN}**: The value can be parsed according to `{PATTERN}`, which `MUST` follow the date formatting syntax of C / Python [strftime](http://strftime.org/).", - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `datetime` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"timestamp\",\n \"type\": \"datetime\"\n}\n", - "{\n \"name\": \"timestamp\",\n \"type\": \"datetime\",\n \"format\": \"default\"\n}\n" - ] - }, - { - "type": "object", - "title": "Year Field", - "description": "A calendar year, being an integer with 4 digits. Equivalent to [gYear in XML Schema](https://www.w3.org/TR/xmlschema-2/#gYear)", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `year`.", - "enum": [ - "year" - ] - }, - "format": { - "description": "There are no format keyword options for `year`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `year` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "integer" - } - } - ] - }, - "minimum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "maximum": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"year\",\n \"type\": \"year\"\n}\n", - "{\n \"name\": \"year\",\n \"type\": \"year\",\n \"constraints\": {\n \"minimum\": 1970,\n \"maximum\": 2003\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Year Month Field", - "description": "A calendar year month, being an integer with 1 or 2 digits. Equivalent to [gYearMonth in XML Schema](https://www.w3.org/TR/xmlschema-2/#gYearMonth)", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `yearmonth`.", - "enum": [ - "yearmonth" - ] - }, - "format": { - "description": "There are no format keyword options for `yearmonth`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `yearmonth` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"month\",\n \"type\": \"yearmonth\"\n}\n", - "{\n \"name\": \"month\",\n \"type\": \"yearmonth\",\n \"constraints\": {\n \"minimum\": 1,\n \"maximum\": 6\n }\n}\n" - ] - }, - { - "type": "object", - "title": "Boolean Field", - "description": "The field contains boolean (true/false) data.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `boolean`.", - "enum": [ - "boolean" - ] - }, - "format": { - "description": "There are no format keyword options for `boolean`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "trueValues": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - }, - "default": [ - "true", - "True", - "TRUE", - "1" - ] - }, - "falseValues": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - }, - "default": [ - "false", - "False", - "FALSE", - "0" - ] - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `boolean` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "boolean" - } - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"registered\",\n \"type\": \"boolean\"\n}\n" - ] - }, - { - "type": "object", - "title": "Object Field", - "description": "The field contains data which can be parsed as a valid JSON object.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `object`.", - "enum": [ - "object" - ] - }, - "format": { - "description": "There are no format keyword options for `object`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply for `object` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"extra\"\n \"type\": \"object\"\n}\n" - ] - }, - { - "type": "object", - "title": "GeoPoint Field", - "description": "The field contains data describing a geographic point.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `geopoint`.", - "enum": [ - "geopoint" - ] - }, - "format": { - "description": "The format keyword options for `geopoint` are `default`,`array`, and `object`.", - "context": "The following `format` options are supported:\n * **default**: A string of the pattern 'lon, lat', where `lon` is the longitude and `lat` is the latitude.\n * **array**: An array of exactly two items, where each item is either a number, or a string parsable as a number, and the first item is `lon` and the second item is `lat`.\n * **object**: A JSON object with exactly two keys, `lat` and `lon`", - "notes": [ - "Implementations `MUST` strip all white space in the default format of `lon, lat`." - ], - "enum": [ - "default", - "array", - "object" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `geopoint` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "array" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"post_office\",\n \"type\": \"geopoint\"\n}\n", - "{\n \"name\": \"post_office\",\n \"type\": \"geopoint\",\n \"format\": \"array\"\n}\n" - ] - }, - { - "type": "object", - "title": "GeoJSON Field", - "description": "The field contains a JSON object according to GeoJSON or TopoJSON", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `geojson`.", - "enum": [ - "geojson" - ] - }, - "format": { - "description": "The format keyword options for `geojson` are `default` and `topojson`.", - "context": "The following `format` options are supported:\n * **default**: A geojson object as per the [GeoJSON spec](http://geojson.org/).\n * **topojson**: A topojson object as per the [TopoJSON spec](https://github.com/topojson/topojson-specification/blob/master/README.md)", - "enum": [ - "default", - "topojson" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `geojson` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "object" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"city_limits\",\n \"type\": \"geojson\"\n}\n", - "{\n \"name\": \"city_limits\",\n \"type\": \"geojson\",\n \"format\": \"topojson\"\n}\n" - ] - }, - { - "type": "object", - "title": "Array Field", - "description": "The field contains data which can be parsed as a valid JSON array.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `array`.", - "enum": [ - "array" - ] - }, - "format": { - "description": "There are no format keyword options for `array`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply for `array` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "array" - } - } - ] - }, - "minLength": { - "type": "integer", - "description": "An integer that specifies the minimum length of a value." - }, - "maxLength": { - "type": "integer", - "description": "An integer that specifies the maximum length of a value." - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"options\"\n \"type\": \"array\"\n}\n" - ] - }, - { - "type": "object", - "title": "Duration Field", - "description": "The field contains a duration of time.", - "context": "The lexical representation for duration is the [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) extended format `PnYnMnDTnHnMnS`, where `nY` represents the number of years, `nM` the number of months, `nD` the number of days, 'T' is the date/time separator, `nH` the number of hours, `nM` the number of minutes and `nS` the number of seconds. The number of seconds can include decimal digits to arbitrary precision. Date and time elements including their designator may be omitted if their value is zero, and lower order elements may also be omitted for reduced precision. Here we follow the definition of [XML Schema duration datatype](http://www.w3.org/TR/xmlschema-2/#duration) directly and that definition is implicitly inlined here.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `duration`.", - "enum": [ - "duration" - ] - }, - "format": { - "description": "There are no format keyword options for `duration`: only `default` is allowed.", - "enum": [ - "default" - ], - "default": "default" - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints are supported for `duration` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "minimum": { - "type": "string" - }, - "maximum": { - "type": "string" - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"period\"\n \"type\": \"duration\"\n}\n" - ] - }, - { - "type": "object", - "title": "Any Field", - "description": "Any value is accepted, including values that are not captured by the type/format/constraint requirements of the specification.", - "required": [ - "name", - "type" - ], - "properties": { - "name": { - "title": "Name", - "description": "A name for this field.", - "type": "string" - }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" - ] - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ] - }, - "example": { - "title": "Example", - "description": "An example value for the field.", - "type": "string", - "examples": [ - "{\n \"example\": \"Put here an example value for your field\"\n}\n" - ] - }, - "type": { - "description": "The type keyword, which `MUST` be a value of `any`.", - "enum": [ - "any" - ] - }, - "constraints": { - "title": "Constraints", - "description": "The following constraints apply to `any` fields.", - "type": "object", - "properties": { - "required": { - "type": "boolean", - "description": "Indicates whether a property must have a value for each instance.", - "context": "An empty string is considered to be a missing value." - }, - "unique": { - "type": "boolean", - "description": "When `true`, each value for the property `MUST` be unique." - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - } - } - }, - "rdfType": { - "type": "string", - "description": "The RDF type for this field." - } - }, - "examples": [ - "{\n \"name\": \"notes\",\n \"type\": \"any\"\n" - ] - } - ] - }, - "description": "An `array` of Table Schema Field objects.", - "examples": [ - "{\n \"fields\": [\n {\n \"name\": \"my-field-name\"\n }\n ]\n}\n", - "{\n \"fields\": [\n {\n \"name\": \"my-field-name\",\n \"type\": \"number\"\n },\n {\n \"name\": \"my-field-name-2\",\n \"type\": \"string\",\n \"format\": \"email\"\n }\n ]\n}\n" - ] - }, - "primaryKey": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "items": { - "type": "string" - } - }, - { - "type": "string" - } - ], - "description": "A primary key is a field name or an array of field names, whose values `MUST` uniquely identify each row in the table.", - "context": "Field name in the `primaryKey` `MUST` be unique, and `MUST` match a field name in the associated table. It is acceptable to have an array with a single value, indicating that the value of a single field is the primary key.", - "examples": [ - "{\n \"primaryKey\": [\n \"name\"\n ]\n}\n", - "{\n \"primaryKey\": [\n \"first_name\",\n \"last_name\"\n ]\n}\n" - ] - }, - "foreignKeys": { - "type": "array", - "minItems": 1, - "items": { - "title": "Table Schema Foreign Key", - "description": "Table Schema Foreign Key", - "type": "object", - "required": [ - "fields", - "reference" - ], - "oneOf": [ - { - "properties": { - "fields": { - "type": "array", - "items": { - "type": "string", - "minItems": 1, - "uniqueItems": true, - "description": "Fields that make up the primary key." - } - }, - "reference": { - "type": "object", - "required": [ - "resource", - "fields" - ], - "properties": { - "resource": { - "type": "string", - "default": "" - }, - "fields": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - } - } - } - } - }, - { - "properties": { - "fields": { - "type": "string", - "description": "Fields that make up the primary key." - }, - "reference": { - "type": "object", - "required": [ - "resource", - "fields" - ], - "properties": { - "resource": { - "type": "string", - "default": "" - }, - "fields": { - "type": "string" - } - } - } - } - } - ] - }, - "examples": [ - "{\n \"foreignKeys\": [\n {\n \"fields\": \"state\",\n \"reference\": {\n \"resource\": \"the-resource\",\n \"fields\": \"state_id\"\n }\n }\n ]\n}\n", - "{\n \"foreignKeys\": [\n {\n \"fields\": \"state\",\n \"reference\": {\n \"resource\": \"\",\n \"fields\": \"id\"\n }\n }\n ]\n}\n" - ] - }, - "missingValues": { - "type": "array", - "items": { - "type": "string" - }, - "default": [ - "" - ], - "description": "Values that when encountered in the source, should be considered as `null`, 'not present', or 'blank' values.", - "context": "Many datasets arrive with missing data values, either because a value was not collected or it never existed.\nMissing values may be indicated simply by the value being empty in other cases a special value may have been used e.g. `-`, `NaN`, `0`, `-9999` etc.\nThe `missingValues` property provides a way to indicate that these values should be interpreted as equivalent to null.\n\n`missingValues` are strings rather than being the data type of the particular field. This allows for comparison prior to casting and for fields to have missing value which are not of their type, for example a `number` field to have missing values indicated by `-`.\n\nThe default value of `missingValue` for a non-string type field is the empty string `''`. For string type fields there is no default for `missingValue` (for string fields the empty string `''` is a valid value and need not indicate null).", - "examples": [ - "{\n \"missingValues\": [\n \"-\",\n \"NaN\",\n \"\"\n ]\n}\n", - "{\n \"missingValues\": []\n}\n" - ] - } - }, - "examples": [ - "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" - ] -} \ No newline at end of file diff --git a/tests/test_all b/tests/test_all deleted file mode 100755 index cfd886da..00000000 --- a/tests/test_all +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Runs pre-commit, then all the scripts in this dir, and then builds documentaiton locally - -# Get the list of script files in the directory -script_files=("$(dirname "$(realpath "$0")")"/*) -num_files=$((${#script_files[@]} + 1)) -test_num=1 -error_scripts=() - -echo "Found $num_files tests" - -echo "----- Running test 1 of $num_files: pre-commit" -pre-commit run --all-files - -# Iterate over script files in the directory -for script_file in "${script_files[@]}"; do - - # Exclude the current script and non-executable files from execution - if [ "$(basename "$script_file")" != "test_all" ]; then - ((test_num++)) - if [ -x "$script_file" ]; then - # Update progress - echo "----- Running test $test_num of $num_files: $script_file" - - # Execute the script and capture errors if there are any - if output=$("$script_file" 2>&1); then - echo ":-) $script_file ran successfully." - else - echo "!!! $script_file encountered an error." - error_scripts+=("$script_file") - echo "$output" >&2 - fi - else - # Notify non-executable files - echo "!!! File is not executable so was not run: $script_file." - echo - fi - fi -done -echo "----- Running test $num_files/$num_files: Building documentation locally" -mkdocs build --quiet - -if [ ${#error_scripts[@]} -gt 0 ]; then - echo "The following scripts executed with errors:" - for script in "${error_scripts[@]}"; do - echo "$script" - done -fi diff --git a/tests/test_local_spec b/tests/test_local_spec new file mode 100755 index 00000000..195bf940 --- /dev/null +++ b/tests/test_local_spec @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +description="Validates the local spec and documentation." +usage="tests/test_local_spec" + +################################################################################ +# Run Pre-Commit # +################################################################################ +echo "Running pre-commit." +for _ in {1..2}; do + if output=$(pre-commit run --all-files 2>&1); then + echo ":-) pre-commit ran successfully on try $_." + break + fi +done + +################################################################################ +# Run Tests # +################################################################################ +echo "Running tests for local compatibility." + +# Function to execute the script and capture errors if there are any +execute_script() { + script=$1 + shift + echo "--- Running $script $@ ---" + if output=$("$script" "$@" 2>&1); then + echo ":-) $script ran successfully." + else + echo "\033[31m!!! $script encountered an error.\033[0m" + echo "$output" >&2 + error_scripts+=("$script") + fi +} + +error_scripts=() +# Execute the scripts +execute_script "tests/validate_profile" +execute_script "tests/validate_table_schemas" + +################################################################################ +# Build Docs Locally # +################################################################################ +echo "Building docs locally." +if output=$(mkdocs build --quiet 2>&1); then + echo ":-) Docs built successfully." +else + echo "\033[31m!!! mkdocs encountered an error.\033[0m" + echo "$output" >&2 + error_scripts+=("mkdocs build --quiet") +fi + +################################################################################ +# Summarize Errors # +################################################################################ +if [ ${#error_scripts[@]} -gt 0 ]; then + echo "The following scripts executed with errors:" + for script in "${error_scripts[@]}"; do + echo "$script" + done +fi diff --git a/tests/test_samples_to_canonical b/tests/test_samples_to_canonical new file mode 100755 index 00000000..0bfc5ac9 --- /dev/null +++ b/tests/test_samples_to_canonical @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +description="Run test to validate local samples to canonical remote spec" +usage="tests/test_samples_to_canonical" + +################################################################################ +# Specify Scripts and Parameters # +################################################################################ +test_dir=$(dirname "$0") +script="$test_dir/validate_samples" +args="-R" + +################################################################################ +# Run Test # +################################################################################ + +echo "Running the test to validate local samples to canonical remote spec" + +# Execute the script and capture errors if there are any +if output=$("$script" $args 2>&1); then + echo ":-) $script ran successfully." +else + printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" + echo "$output" >&2 +fi \ No newline at end of file diff --git a/tests/test_samples_to_local b/tests/test_samples_to_local new file mode 100755 index 00000000..57041bf8 --- /dev/null +++ b/tests/test_samples_to_local @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +description="Run test to validate local samples to local remote spec" +usage="tests/test_samples_to_local" + +################################################################################ +# Specify Scripts and Parameters # +################################################################################ +test_dir=$(dirname "$0") +script="$test_dir/validate_samples" +args="-L" + +################################################################################ +# Run Test # +################################################################################ + +echo "Running the test to validate local samples to local spec" + +# Execute the script and capture errors if there are any +if output=$("$script" $args 2>&1); then + echo ":-) $script ran successfully." +else + printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" + echo "$output" >&2 +fi \ No newline at end of file diff --git a/tests/validate_profile b/tests/validate_profile index 81f4ddd1..2c200be0 100755 --- a/tests/validate_profile +++ b/tests/validate_profile @@ -1,17 +1,24 @@ #!/usr/bin/env bash -echo "Validating tides-data-package.json profile" +# Validate "tides-datapackage-profile.json" against the Frictionless Data Package schema. +# The script downloads the Data Packaage schema file to a temporary directory using curl and +# then uses the jsonschema-cli tool to perform the validation. + +# Usage: ./tests/validate_profile +# Note: should be run from the base directory of the repo + +echo "--- Validating tides-datapackage-profile.json ---" # Set the temporary directory path temp_dir=$(mktemp -d) -# Download the schema file to the temporary directory +# Download the data package schema file to the temporary directory schema_url="https://specs.frictionlessdata.io/schemas/data-package.json" schema_file="$temp_dir/data-package.json" curl -o "$schema_file" "$schema_url" -# Validate datapackage against the downloaded schema -datapackage_file="spec/tides-data.json" +echo "Validate tides-datapackage-profile against the downloaded schema" +datapackage_file="spec/tides-datapackage-profile.json" jsonschema-cli validate "$schema_file" "$datapackage_file" # Clean up the temporary directory diff --git a/tests/validate_samples b/tests/validate_samples index cac8c226..c628838a 100755 --- a/tests/validate_samples +++ b/tests/validate_samples @@ -1,8 +1,100 @@ #!/usr/bin/env bash -echo "Validating sample data" + +description="Validate all the TIDES data in the samples directory" + +usage=" +Usage: bin/validate_samples.sh [-R] [-L] [-r remote_spec_ref] [-l local_spec_ref] + -R: Optional. Use the default remote profile ref (DEFAULT_REMOTE_REF, which is set to "main"). + -L: Optional. Use the default local spec path (DEFAULT_LOCAL_SPEC), which is set to "spec".. + -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile where the profile is in the sub-path /spec/tides-data-package.json. + Should not be used with -l option. Example: -r main. + -l local_spec_path: Optional. Specify the location of the local tides-data-package-json to use. + Should not be used with -r option. Example: -l spec. +" + +example_usage="tests/validate_samples -r main" + +################################################################################ +# Default Parameters # +################################################################################ +DEFAULT_REMOTE_REF="main" +DEFAULT_LOCAL_SPEC="spec" + +################################################################################ +# Help # +################################################################################ + +# Display help message +function display_help() { + echo "$description" + echo "$usage" + echo "$example_usage" +} + +# Check for help flag +if [ "$1" == "--help" ]; then + display_help + exit 0 +fi + +################################################################################ +# Source utility functions # +################################################################################ + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bin_path="$script_dir/../bin" + +################################################################################ +# Process the input options. # +################################################################################ +remote_spec_ref="" +local_spec_path="" +use_default_remote=false +use_default_local=false + +# Parse command-line arguments +while getopts "RLr:l:" opt; do + case $opt in + R) + use_default_remote=true + ;; + L) + use_default_local=true + ;; + r) + remote_spec_ref="$OPTARG" + ;; + l) + local_spec_path="$OPTARG" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + display_help + exit 1 + ;; + esac +done + +############################################################################### +# Main # +################################################################################ + +echo "$description" + +# Set default values if requested +if [ "$use_default_local" = true ]; then + local_spec_path="$DEFAULT_LOCAL_SPEC" +fi + +if [ "$use_default_remote" = true ]; then + remote_spec_ref="$DEFAULT_REMOTE_REF" +fi + shopt -s nullglob for file in samples/*/TIDES/datapackage.json; do - echo "Validating: $file" - frictionless validate --schema-sync $file + sample_dir=$(echo "$file" | sed 's|/datapackage.json$||') + echo "...validating $sample_dir" + "$bin_path/validate-datapackage" -d "$sample_dir" -r "$remote_spec_ref" -l "$local_spec_path" done shopt -u nullglob From ad10ddef1671b5649af1905942db5fe282e0a3a8 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 21 Jul 2023 18:13:21 -0700 Subject: [PATCH 04/75] Add check for if frictionless is installed --- bin/utils | 10 ++++++++++ bin/validate-datapackage | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/bin/utils b/bin/utils index ebc4dc7a..75f5a05a 100755 --- a/bin/utils +++ b/bin/utils @@ -10,6 +10,16 @@ check_jsonschema-cli() { fi } +# Check if frictionless is installed +check_frictionless() { + if ! command -v frictionless >/dev/null 2>&1; then + echo >&2 "\033[31m!!! frictionless is required but not found.\033[0m + + You can install it using 'pip install frictionless'. Aborting." + exit 1 + fi +} + # Function to check jq and provide installation instructions if not found check_jq() { if ! command -v jq >/dev/null 2>&1; then diff --git a/bin/validate-datapackage b/bin/validate-datapackage index 5d940d6a..a6fbbcff 100755 --- a/bin/validate-datapackage +++ b/bin/validate-datapackage @@ -43,6 +43,11 @@ echo "$description" source "$(dirname "${BASH_SOURCE[0]}")/utils" script_dir=$(dirname "$0") +################################################################################ +# Check Requirements # +################################################################################ +check_frictionless + ################################################################################ # Process the parameters + input options. # ################################################################################ From 45c116a082d5226bf4a4bb03bb95f988fa58168b Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 21 Jul 2023 18:23:23 -0700 Subject: [PATCH 05/75] add documentation about validate-datapackage and validate-datapackage-to-profile --- samples/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/samples/README.md b/samples/README.md index 09b51056..66dc9564 100644 --- a/samples/README.md +++ b/samples/README.md @@ -36,6 +36,26 @@ pip install frictionless frictionless validate --schema-sync path/to/your/datapackage.json ``` +Alternatively, we also provide a wrapper script to provide some additional flexibility and options. + +Usage: `bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path]` + +- `-r remote_spec_ref`: Optional. Specify the ref name of the GitHub repository for validating agianst + a remote profile. Should not be used with -l option. Example: `-r main` +- `-l local_spec_path`: Optional. Specify the path of the local schema directory. + Default is 'spec'. Is only used if remote_spec_ref = local. +- `-d dataset_path`: Optional. Specify the path of the TIDES datapackage.json. + Default is the current directory. + +Key usage examples: + +- `bin/validate-datapackage -l spec -d samples//TIDES`: Validate my sample data to a version of the spec located in the `/spec` directory. +- `bin/validate-datapackage -r main -d samples//TIDES`: Validate my sample data to the canonical version of the TIDES spec. +- `bin/validate-datapackage -r v1.0 -d samples//TIDES`: Validate my sample data to v1.0 of the spec. +- `bin/validate-datapackage -r develop -d samples//TIDES`: Validate my sample data to the current `develop` branch of the TIDES spc. + +If you **only** want to validate your `datapackage.json` file and not the datapackage as a whole, you can run the script `bin/validate-datapackage-to-profile` instead with the same options. Note that this is also run as a part of `validate-datapackage`. + ### Specific files Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: From eee0fd9e1048739645bb6ace9638a8939a31a198 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 24 Jul 2023 12:33:10 -0700 Subject: [PATCH 06/75] old code removed Superceded. --- mkdocs.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index a818df60..644883c3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,11 +35,6 @@ plugins: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' - - redirects: - redirect_maps: #original relative to /docs : redirect target - '../CODE_OF_CONDUCT.md': 'CODE_OF_CONDUCT.md' - '../CONTRIBUTING.md': 'CONTRIBUTING.md' - '../contributors.md': 'contributors.md' - search From a280001e997a55a01a4a870fd2d3b5f4df5a2923 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 24 Jul 2023 13:04:26 -0700 Subject: [PATCH 07/75] Update README to be consistent --- README.md | 34 ++++++++++++++++------------------ docs/datapackage.md | 28 ++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index a6e372ec..5f0dec1d 100644 --- a/README.md +++ b/README.md @@ -12,35 +12,33 @@ Human-friendlier documentation is auto-generated and available at: - [Architecture](https://tides-transit.github.io/TIDES/main/architecture) - [Table Schemas](https://tides-transit.github.io/TIDES/main/tables) -## Example Data +### Data Package -Sample data can be found in the `/samples` directory, with one directory for each example. +Directories with TIDES data must contain metadata in a [`datapackage.json`](https://tides-transit.github.io/TIDES/main/datapackage) file in a format compliant with the [`tides-datapackage-profile`](https://tides-transit.github.io/TIDES/main/datapackage) of a [`frictionless data package`](https://specs.frictionlessdata.io/data-package/). -## Validating TIDES data +[`/samples/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/datapackage.json) has a template datapackage which can be used. -TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: +## Sample Data -```bash -pip install frictionless -frictionless validate path/to/your/datapackage.json -``` +[Sample data](https://tides-transit.github.io/TIDES/main/samples) can be found in the `/samples` directory, with one directory for each sample. -### Data Package +### Template -To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/datapackage.json). +Templates of `datapackage.json` and each TIDES file type are located in the `/samples/template` directory. -Once this is created, mapping the data files to the schema, simply run: +## Validating TIDES data -```sh -frictionless validate datapackage.json -``` +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: -### Specific files +```bash +pip install frictionless +frictionless validate --schema-sync path/to/your/datapackage.json +``` -Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: +Several other validation scripts and tools with more flexibility such as validating to the canonical, named version or a local spec can be found in the `/bin` directory, with usage available with the `--help` flag. -```sh -frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +```bash +bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_path] ``` ## Contributing to TIDES diff --git a/docs/datapackage.md b/docs/datapackage.md index 63025fed..c6cb04ec 100644 --- a/docs/datapackage.md +++ b/docs/datapackage.md @@ -32,22 +32,38 @@ There are lots of options for validating your `datapackage.json` file including: ### CLI -You can easily validate your data package file with the script provided in [`/bin/validate-data-package-json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/bin/validate-data-package-json) +You can easily validate your data package file with the script provided in [`/bin/validate-data-package-to-profile`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/bin/validate-data-package-to-profile) ??? tip "installation requirements" Make sure you have jsonschema-cli installed. You can install it specifically or with all of the other suggested tools using one of the commands below: - ```sh - pip install jsonschema-cli - pip install -r requirements.txt - ``` + === "Mac" + + ```sh + brew install jq # mac + pip install -r requirements.txt + ``` + + === "Windows" + + ```sh + curl -L -o /usr/bin/jq.exe https://github.com/stedolan/jq/releases/latest/download/jq-win64.exe #windows + pip install -r requirements.txt + ``` + + === "Linux" + + ```sh + sudo apt-get install jq # linux + pip install -r requirements.txt + ``` ```sh title="usage" validate-data-package-json -f my-datapackage.json ``` -{{ include_file('bin/validate-data-package-json',code_type='sh') }} +{{ include_file('bin/validate-datapackage-to-profile',code_type='sh') }} ### Point-and-Drool From 0223c9f6d8189077122811f494fe8b5cbfc87daf Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:36:02 -0500 Subject: [PATCH 08/75] Try capturing results JSON more predictably --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 6e05dc2e..4d05a0bc 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,7 +37,7 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) - results+="$result\n" + results+="$result\n\n" done echo "{name}=${results}" >> $GITHUB_ENV - name: Comment on PR @@ -47,7 +47,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const samples = '${{ env.samples }}'.trim().split('\n'); - const results = '${{ env.results }}'.trim().split('\n'); + const results = '${{ env.results }}'.trim().split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From 1b594aa5a947ecbb785781567206b5ade137b8cf Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:40:47 -0500 Subject: [PATCH 09/75] Trim after split --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 4d05a0bc..2d35f997 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -46,8 +46,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const samples = '${{ env.samples }}'.trim().split('\n'); - const results = '${{ env.results }}'.trim().split('\n\n'); + const samples = '${{ env.samples }}'.split('\n').trim().; + const results = '${{ env.results }}'.split('\n\n').trim(); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From c257a7a1f853fa083e5b7f04e861d0c419dc0015 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:41:53 -0500 Subject: [PATCH 10/75] Fix trim() call --- .github/workflows/validate_samples.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 2d35f997..1ae2e8ca 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -46,14 +46,14 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const samples = '${{ env.samples }}'.split('\n').trim().; - const results = '${{ env.results }}'.split('\n\n').trim(); + const samples = '${{ env.samples }}'.split('\n'); + const results = '${{ env.results }}'.split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; for (let i = 0; i < samples.length; i++) { const sample = samples[i]; - const result = JSON.parse(results[i]); + const result = JSON.parse(results[i].trim()); let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From d6030a4ff2181b66da7de6ccd1d1b04c66ccf57d Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:43:25 -0500 Subject: [PATCH 11/75] Remove trim altogether --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 1ae2e8ca..4f965dc0 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -53,7 +53,7 @@ jobs: comment += '| ------ | ------ |\n'; for (let i = 0; i < samples.length; i++) { const sample = samples[i]; - const result = JSON.parse(results[i].trim()); + const result = JSON.parse(results[i]); let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From ab174929e758dbc0e565e34129b685129b3e7d27 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:47:01 -0500 Subject: [PATCH 12/75] Log before presumed failure point --- .github/workflows/validate_samples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 4f965dc0..3bf2df94 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -52,6 +52,8 @@ jobs: comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; for (let i = 0; i < samples.length; i++) { + console.log(samples[i]) + console.log(results[i]) const sample = samples[i]; const result = JSON.parse(results[i]); let status = ''; From e30fefb1de4a8d95b0d7e9f8556ba4ef9252001c Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:50:29 -0500 Subject: [PATCH 13/75] Additional logging --- .github/workflows/validate_samples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 3bf2df94..fc257dff 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -46,11 +46,13 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | + console.log("Begin sample construction") const samples = '${{ env.samples }}'.split('\n'); const results = '${{ env.results }}'.split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; + console.log("Constructed header") for (let i = 0; i < samples.length; i++) { console.log(samples[i]) console.log(results[i]) From 3318e80c3dd6c3796f25dbda803f480a0a21e9a1 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:53:39 -0500 Subject: [PATCH 14/75] Confirm inaccurate passage suspicion --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index fc257dff..102e94c7 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -54,8 +54,8 @@ jobs: comment += '| ------ | ------ |\n'; console.log("Constructed header") for (let i = 0; i < samples.length; i++) { - console.log(samples[i]) - console.log(results[i]) + console.log(samples.length) + console.log(results.length) const sample = samples[i]; const result = JSON.parse(results[i]); let status = ''; From 888958a9f86ffe736a54f60aaf62b5dff3b732b9 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:56:33 -0500 Subject: [PATCH 15/75] Some extra confirmation logging --- .github/workflows/validate_samples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 102e94c7..d64e3ae9 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,7 +35,9 @@ jobs: run: | samples=(${{ steps.sample_folders.outputs.samples }}) results="" + console.log(samples.length) for sample in "${samples[@]}"; do + console.log(sample) result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) results+="$result\n\n" done From 9cfdbe761a08099f5928ff2d0a86f980d80db266 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:57:52 -0500 Subject: [PATCH 16/75] Log in the correct place --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index d64e3ae9..617a6866 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -34,8 +34,8 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - results="" console.log(samples.length) + results="" for sample in "${samples[@]}"; do console.log(sample) result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) From ab1c13a33c3b5539e0c944d3ba64d0d5ac5b70f2 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 16:58:53 -0500 Subject: [PATCH 17/75] Log full text of samples variable --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 617a6866..ce3c8a1c 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -34,7 +34,7 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - console.log(samples.length) + console.log(samples) results="" for sample in "${samples[@]}"; do console.log(sample) From 5a9f22fba5f7f221a081993e0d257af54f845395 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:01:19 -0500 Subject: [PATCH 18/75] Move logging --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index ce3c8a1c..7a590f37 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -29,12 +29,12 @@ jobs: run: | pwd samples=$(find ./samples -type d -name 'TIDES') + console.log(samples) echo "{name}={value}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - console.log(samples) results="" for sample in "${samples[@]}"; do console.log(sample) From 8519cb683cf473affb4160ec0f906b84ab4d7495 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:02:51 -0500 Subject: [PATCH 19/75] Echo instead of log --- .github/workflows/validate_samples.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 7a590f37..29758f7a 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -29,15 +29,14 @@ jobs: run: | pwd samples=$(find ./samples -type d -name 'TIDES') - console.log(samples) echo "{name}={value}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) + echo samples results="" for sample in "${samples[@]}"; do - console.log(sample) result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) results+="$result\n\n" done From ddea99299caba135b2f0a43c847b70323b3bee1f Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:04:11 -0500 Subject: [PATCH 20/75] Echo variable, not string literal --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 29758f7a..9c3b9141 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -34,7 +34,7 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - echo samples + echo $samples results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) From 1d58646fb0a2ac628d104cfb287c143a0450cbe9 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:06:58 -0500 Subject: [PATCH 21/75] Echo in previous task, just to be sure --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 9c3b9141..50336f65 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -29,6 +29,7 @@ jobs: run: | pwd samples=$(find ./samples -type d -name 'TIDES') + echo $samples echo "{name}={value}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation From 0351cf350d49c99a8e7134c0d17b4cbbac188dbe Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:11:48 -0500 Subject: [PATCH 22/75] Try pushing different name --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 50336f65..b28a0d71 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -30,7 +30,7 @@ jobs: pwd samples=$(find ./samples -type d -name 'TIDES') echo $samples - echo "{name}={value}" >> $GITHUB_OUTPUT + echo "{name}={samples}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | From 87a77233118a56cd8ce9cf090e05ac62f5a85405 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Thu, 24 Aug 2023 17:13:29 -0500 Subject: [PATCH 23/75] Ref as variable --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index b28a0d71..ee4d1bc0 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -30,7 +30,7 @@ jobs: pwd samples=$(find ./samples -type d -name 'TIDES') echo $samples - echo "{name}={samples}" >> $GITHUB_OUTPUT + echo "{name}=${samples}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | From 1174d8414254b72f409af4520b3f63e42e975d83 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 09:50:30 -0500 Subject: [PATCH 24/75] Explicitly set output variable names --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index ee4d1bc0..7ad6f06e 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -30,7 +30,7 @@ jobs: pwd samples=$(find ./samples -type d -name 'TIDES') echo $samples - echo "{name}=${samples}" >> $GITHUB_OUTPUT + echo "samples=${samples}" >> $GITHUB_OUTPUT - name: Validate Sample Data id: validation run: | @@ -41,7 +41,7 @@ jobs: result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) results+="$result\n\n" done - echo "{name}=${results}" >> $GITHUB_ENV + echo "results=${results}" >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From d3a0a07a61ebd24a3c9f759f77ed6a4ed98c6238 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 09:53:34 -0500 Subject: [PATCH 25/75] Additional echo for diagnostic purposes --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 7ad6f06e..241d2917 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,6 +38,7 @@ jobs: echo $samples results="" for sample in "${samples[@]}"; do + echo $sample result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) results+="$result\n\n" done From 93c169ff7ca61757ec56465aa00163a01866efb4 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 09:55:41 -0500 Subject: [PATCH 26/75] Try setting file destination differently --- .github/workflows/validate_samples.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 241d2917..03fc43e0 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,11 +35,9 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - echo $samples results="" for sample in "${samples[@]}"; do - echo $sample - result=$(frictionless validate --schema-sync "$sample/datapackage.json" --json) + result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done echo "results=${results}" >> $GITHUB_ENV From 16cdd4be1aa10a3c74121e61b41921484e61e81f Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 09:57:04 -0500 Subject: [PATCH 27/75] Echo file reference --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 03fc43e0..96221985 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,6 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do + echo ${sample}/datapackage.json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done From 9b6dee653dc4e6cfdd35e9bb499caf6d162fd578 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 09:59:27 -0500 Subject: [PATCH 28/75] Experiment with upgraded attrs install --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 96221985..f4165007 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -24,6 +24,8 @@ jobs: python-version: 3.x - name: Install Frictionless run: pip install frictionless[excel,json]>=5.0.0 + - name: Experimentally upgrade attrs + run: pip install attrs --upgrade - name: Find Sample Folders id: sample_folders run: | @@ -37,7 +39,6 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - echo ${sample}/datapackage.json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done From 716b612eb759215a297b6fbfc776078b728a1d81 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 10:01:36 -0500 Subject: [PATCH 29/75] Call validation directly to surface detailed error --- .github/workflows/validate_samples.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index f4165007..e184633c 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -24,8 +24,6 @@ jobs: python-version: 3.x - name: Install Frictionless run: pip install frictionless[excel,json]>=5.0.0 - - name: Experimentally upgrade attrs - run: pip install attrs --upgrade - name: Find Sample Folders id: sample_folders run: | @@ -39,6 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do + frictionless validate --schema-sync "${sample}/datapackage.json" --json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done From 0d3d28d2ff25f6df319e1d6ab257a1ca0e91c215 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 10:07:11 -0500 Subject: [PATCH 30/75] Substitute development files for testing purposes --- samples/template/TIDES/datapackage.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index dd0e5cca..b3c85fc3 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -3,7 +3,7 @@ "title": "Template TIDES Data Package Example", "agency": "Transit Agency Name", "ntd_id": "1234-56", - "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", + "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/tides-datapackage-profile.json", "licenses": [ { "name": "Apache-2.0" @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", From 08b5a7684c41d3b0e0192559313c33d4b2ef904e Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 10:09:09 -0500 Subject: [PATCH 31/75] tree -> blob --- samples/template/TIDES/datapackage.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index b3c85fc3..0918d3ad 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -3,7 +3,7 @@ "title": "Template TIDES Data Package Example", "agency": "Transit Agency Name", "ntd_id": "1234-56", - "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/tides-datapackage-profile.json", + "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/tides-datapackage-profile.json", "licenses": [ { "name": "Apache-2.0" @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/tree/sample-data-doc/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", From ae1a08ae4dac80d012e58f47eaafe8f6ee912632 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 10:11:40 -0500 Subject: [PATCH 32/75] Raw content ref --- samples/template/TIDES/datapackage.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index 0918d3ad..c667a37a 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -3,7 +3,7 @@ "title": "Template TIDES Data Package Example", "agency": "Transit Agency Name", "ntd_id": "1234-56", - "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/tides-datapackage-profile.json", + "profile": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json", "licenses": [ { "name": "Apache-2.0" @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/blob/sample-data-doc/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", From 125739bd9108ca0f17198b02c38c59e31696955e Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:30:56 -0500 Subject: [PATCH 33/75] Multi-line string env output from action --- .github/workflows/validate_samples.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index e184633c..27d44313 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,11 +37,12 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - frictionless validate --schema-sync "${sample}/datapackage.json" --json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done - echo "results=${results}" >> $GITHUB_ENV + echo 'results<> $GITHUB_ENV + cat $results >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From 27ecbd995c7ef6b197107799cc6c2876ab47a4f6 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:32:51 -0500 Subject: [PATCH 34/75] Added quotes --- .github/workflows/validate_samples.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 27d44313..7f797ba2 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -40,9 +40,9 @@ jobs: result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done - echo 'results<> $GITHUB_ENV - cat $results >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV + echo "results<> $GITHUB_ENV + cat "$results" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From 23d55060b7d4e939b430e8352893a0720bb0cf95 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:37:41 -0500 Subject: [PATCH 35/75] Echo results before assignment --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 7f797ba2..1f4659d4 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -40,8 +40,9 @@ jobs: result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done + echo $results echo "results<> $GITHUB_ENV - cat "$results" >> $GITHUB_ENV + cat $results >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' From 565e9e5cbd0bcc854d2b5b967d982af38e00e001 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:42:22 -0500 Subject: [PATCH 36/75] Use direct env write strategy --- .github/workflows/validate_samples.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 1f4659d4..658b7620 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,14 +35,10 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - results="" + echo "RESULTS<> $GITHUB_ENV for sample in "${samples[@]}"; do - result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) - results+="$result\n\n" + frictionless validate --schema-sync "${sample}/datapackage.json" --json >> $GITHUB_ENV done - echo $results - echo "results<> $GITHUB_ENV - cat $results >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' From 300fa9d81de200f7f97d6c9fb91ef9b9006a5649 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:44:28 -0500 Subject: [PATCH 37/75] Log samples and results --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 658b7620..8a466e8b 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -54,8 +54,8 @@ jobs: comment += '| ------ | ------ |\n'; console.log("Constructed header") for (let i = 0; i < samples.length; i++) { - console.log(samples.length) - console.log(results.length) + console.log(samples[i]) + console.log(results[i]) const sample = samples[i]; const result = JSON.parse(results[i]); let status = ''; From 7e78e83ad4498b1fee1569751ae457e62338e36f Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:46:52 -0500 Subject: [PATCH 38/75] Echo results at end of validation --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 8a466e8b..3cfea86d 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -40,6 +40,7 @@ jobs: frictionless validate --schema-sync "${sample}/datapackage.json" --json >> $GITHUB_ENV done echo "EOF" >> $GITHUB_ENV + echo "$RESULTS" - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From 9a46d52bbf34d6dba06d14376cb0b0929c22f2ec Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:53:15 -0500 Subject: [PATCH 39/75] Try to get correct format for Frictionless output to env --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 3cfea86d..6003a760 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,7 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) echo "RESULTS<> $GITHUB_ENV for sample in "${samples[@]}"; do - frictionless validate --schema-sync "${sample}/datapackage.json" --json >> $GITHUB_ENV + "$(frictionless validate --schema-sync "${sample}/datapackage.json" --json)" >> $GITHUB_ENV done echo "EOF" >> $GITHUB_ENV echo "$RESULTS" From 0f12c009e5eed848b1414212dcfa396dbc9201a1 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:53:26 -0500 Subject: [PATCH 40/75] Indentation fix --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 6003a760..ec1cd445 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,7 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) echo "RESULTS<> $GITHUB_ENV for sample in "${samples[@]}"; do - "$(frictionless validate --schema-sync "${sample}/datapackage.json" --json)" >> $GITHUB_ENV + "$(frictionless validate --schema-sync "${sample}/datapackage.json" --json)" >> $GITHUB_ENV done echo "EOF" >> $GITHUB_ENV echo "$RESULTS" From a64636ac3b9eecb4941cc813b87301dded95d466 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 11:59:46 -0500 Subject: [PATCH 41/75] Different echo format --- .github/workflows/validate_samples.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index ec1cd445..f50294fb 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,12 +35,15 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - echo "RESULTS<> $GITHUB_ENV + results="" for sample in "${samples[@]}"; do - "$(frictionless validate --schema-sync "${sample}/datapackage.json" --json)" >> $GITHUB_ENV + result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) + results+="$result\n\n" done + echo ${results} + echo "RESULTS<> $GITHUB_ENV + cat ${results} >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "$RESULTS" - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 @@ -55,8 +58,8 @@ jobs: comment += '| ------ | ------ |\n'; console.log("Constructed header") for (let i = 0; i < samples.length; i++) { - console.log(samples[i]) - console.log(results[i]) + console.log(samples.length) + console.log(results.length) const sample = samples[i]; const result = JSON.parse(results[i]); let status = ''; From 0d48726ad7e17eff4eb7aea51192f8360fa021ef Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:01:19 -0500 Subject: [PATCH 42/75] Echo instead of cat --- .github/workflows/validate_samples.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index f50294fb..e5d2e49a 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -40,9 +40,8 @@ jobs: result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done - echo ${results} echo "RESULTS<> $GITHUB_ENV - cat ${results} >> $GITHUB_ENV + echo ${results} >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - name: Comment on PR if: github.event_name == 'pull_request' From 45ed9338289eab268a883dcd0586b509433a0db6 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:02:51 -0500 Subject: [PATCH 43/75] Log correct things --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index e5d2e49a..c0ea8496 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -57,8 +57,8 @@ jobs: comment += '| ------ | ------ |\n'; console.log("Constructed header") for (let i = 0; i < samples.length; i++) { - console.log(samples.length) - console.log(results.length) + console.log(samples[i]) + console.log(results[i]) const sample = samples[i]; const result = JSON.parse(results[i]); let status = ''; From 282361d87fb41c5c562b119233c714a6eeef241d Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:04:35 -0500 Subject: [PATCH 44/75] Echo at end of previous task once more --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index c0ea8496..6cb24360 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -43,6 +43,7 @@ jobs: echo "RESULTS<> $GITHUB_ENV echo ${results} >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + echo "$RESULTS" - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From 599e32697e7b031f62b971dc14f8711d38bab8ea Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:08:17 -0500 Subject: [PATCH 45/75] Play around with cat format --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 6cb24360..148d6e78 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -40,8 +40,9 @@ jobs: result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done + echo "RESULTS<> $GITHUB_ENV - echo ${results} >> $GITHUB_ENV + cat $results >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV echo "$RESULTS" - name: Comment on PR From fccedfe36fb6fde0e8b7ece8cc46528de3172865 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:10:52 -0500 Subject: [PATCH 46/75] Check GITHUB_ENV --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 148d6e78..0d912a75 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -42,9 +42,9 @@ jobs: done echo "RESULTS<> $GITHUB_ENV - cat $results >> $GITHUB_ENV + echo ${results} >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "$RESULTS" + echo "$GITHUB_ENV" - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 From ed9814c14f3acddadeee192891872983d60ecb85 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:13:37 -0500 Subject: [PATCH 47/75] Try writing to output instead of env --- .github/workflows/validate_samples.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 0d912a75..979c3946 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -41,10 +41,9 @@ jobs: results+="$result\n\n" done - echo "RESULTS<> $GITHUB_ENV - echo ${results} >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - echo "$GITHUB_ENV" + echo "RESULTS<> $GITHUB_OUTPUT + echo ${results} >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - name: Comment on PR if: github.event_name == 'pull_request' uses: actions/github-script@v4 @@ -52,8 +51,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | console.log("Begin sample construction") - const samples = '${{ env.samples }}'.split('\n'); - const results = '${{ env.results }}'.split('\n\n'); + const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); + const results = '${{ steps.validation.outputs.results }}'.split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From 0b65469d6465c347dda5c1633fb799df40bc9a38 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:25:31 -0500 Subject: [PATCH 48/75] Change most refs back to main --- samples/template/TIDES/datapackage.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index c667a37a..e6aa189a 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", From 38183abc8ffcf948557b54d8587a08526e8cc403 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:32:16 -0500 Subject: [PATCH 49/75] Re-add direct validation to surface new error --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 979c3946..7e4a2e78 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,6 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do + frictionless validate --schema-sync "${sample}/datapackage.json" --json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done From 8e96998ee1a0f8c85b8df633cc0766df2b1cbbee Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:42:03 -0500 Subject: [PATCH 50/75] Attempting to ID detailed error for failed validations --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 7e4a2e78..b2557d9f 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,11 +37,12 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - frictionless validate --schema-sync "${sample}/datapackage.json" --json result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done + echo ${results} + echo "RESULTS<> $GITHUB_OUTPUT echo ${results} >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT From cb925f2c09dbc3b98ca0a8fecab1b4fa8e9e607a Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:46:29 -0500 Subject: [PATCH 51/75] Secondary reference --- spec/tides-datapackage-profile.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/tides-datapackage-profile.json b/spec/tides-datapackage-profile.json index aa25313b..2d1ab9e8 100644 --- a/spec/tides-datapackage-profile.json +++ b/spec/tides-datapackage-profile.json @@ -20,13 +20,13 @@ ], "properties": { "profile": { - "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", + "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json", "propertyOrder": 10, "title": "Profile", "description": "The json-schema profile used to validate this datapackage descriptor. Should be `https://raw.githubusercontent.com/TIDES-transit/TIDES//spec/tides-datapackage-profile.json`. Defaults to `https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`", "format": "uri", "examples": [ - "`https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`" + "`https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json`" ] }, "title": { From 2f8141cccfea853081409f190f035964ce7f33aa Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 12:51:41 -0500 Subject: [PATCH 52/75] Different echo format --- .github/workflows/validate_samples.yml | 2 +- spec/tides-datapackage-profile.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index b2557d9f..05213ac7 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -41,7 +41,7 @@ jobs: results+="$result\n\n" done - echo ${results} + echo "$results" echo "RESULTS<> $GITHUB_OUTPUT echo ${results} >> $GITHUB_OUTPUT diff --git a/spec/tides-datapackage-profile.json b/spec/tides-datapackage-profile.json index 2d1ab9e8..aa25313b 100644 --- a/spec/tides-datapackage-profile.json +++ b/spec/tides-datapackage-profile.json @@ -20,13 +20,13 @@ ], "properties": { "profile": { - "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json", + "default": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json", "propertyOrder": 10, "title": "Profile", "description": "The json-schema profile used to validate this datapackage descriptor. Should be `https://raw.githubusercontent.com/TIDES-transit/TIDES//spec/tides-datapackage-profile.json`. Defaults to `https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`", "format": "uri", "examples": [ - "`https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/tides-datapackage-profile.json`" + "`https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-datapackage-profile.json`" ] }, "title": { From 04bebbe7d0b2caae2a6a09f65bd15878efc1cf9e Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 13:20:34 -0500 Subject: [PATCH 53/75] Troubleshooting validationerror status --- .github/workflows/validate_samples.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 05213ac7..9a0f3586 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,6 +37,8 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do + frictionless validate --schema-sync "${sample}/datapackage.json" --json + echo errored after validation result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done From 6a2bfe1b8713c9a61aaf22814b6ff2e7019c20b8 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 13:23:36 -0500 Subject: [PATCH 54/75] Debug flag --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 9a0f3586..f2908f58 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,7 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - frictionless validate --schema-sync "${sample}/datapackage.json" --json + frictionless validate --schema-sync --debug "${sample}/datapackage.json" --json echo errored after validation result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" From 81e14424c88db511dedcdaffb185da1ba3654621 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 13:52:39 -0500 Subject: [PATCH 55/75] Switch back to dev URL refs --- .github/workflows/validate_samples.yml | 4 ---- samples/template/TIDES/datapackage.json | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index f2908f58..979c3946 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,14 +37,10 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - frictionless validate --schema-sync --debug "${sample}/datapackage.json" --json - echo errored after validation result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) results+="$result\n\n" done - echo "$results" - echo "RESULTS<> $GITHUB_OUTPUT echo ${results} >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index e6aa189a..c667a37a 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", From fd8a7f9fb8e7d48d50b2aa44a9f2931e976e48a4 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 14:06:36 -0500 Subject: [PATCH 56/75] Experiment with updated column list --- samples/template/TIDES/datapackage.json | 22 ++++++++++----------- samples/template/TIDES/passenger_events.csv | 2 +- samples/template/TIDES/stop_visits.csv | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index c667a37a..e6aa189a 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?", diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv index d4ece193..39560719 100644 --- a/samples/template/TIDES/passenger_events.csv +++ b/samples/template/TIDES/passenger_events.csv @@ -1 +1 @@ -passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,trip_stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv index ecc6c3b7..90f34d0d 100644 --- a/samples/template/TIDES/stop_visits.csv +++ b/samples/template/TIDES/stop_visits.csv @@ -1 +1 @@ -date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date +date,trip_id_performed,stop_sequence,trip_stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date From 2312ad702a9dffb9cb8d3f5d11629b957c12db22 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 14:18:12 -0500 Subject: [PATCH 57/75] Pass validation errors to surface JSON successfully --- .github/workflows/validate_samples.yml | 2 +- samples/template/TIDES/passenger_events.csv | 2 +- samples/template/TIDES/stop_visits.csv | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 979c3946..b26f93a8 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -37,7 +37,7 @@ jobs: samples=(${{ steps.sample_folders.outputs.samples }}) results="" for sample in "${samples[@]}"; do - result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) + result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true results+="$result\n\n" done diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv index 39560719..d4ece193 100644 --- a/samples/template/TIDES/passenger_events.csv +++ b/samples/template/TIDES/passenger_events.csv @@ -1 +1 @@ -passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,trip_stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id,service_date,passenger_events,event_timestamp diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv index 90f34d0d..ecc6c3b7 100644 --- a/samples/template/TIDES/stop_visits.csv +++ b/samples/template/TIDES/stop_visits.csv @@ -1 +1 @@ -date,trip_id_performed,stop_sequence,trip_stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship,service_date From 143f3a6de2fb06fd465abaa6cc836e081c207f2a Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 14:43:17 -0500 Subject: [PATCH 58/75] Escape single quotes in results --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index b26f93a8..3ffe8bd7 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,6 +38,7 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true + result="${result//\'/\\\'}" results+="$result\n\n" done From dd10ce049864b95089cf1dada094a52d85a0c218 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 14:55:48 -0500 Subject: [PATCH 59/75] Attempt different quote scheme --- .github/workflows/validate_samples.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 3ffe8bd7..e1653fd6 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,7 +38,6 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true - result="${result//\'/\\\'}" results+="$result\n\n" done @@ -52,8 +51,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | console.log("Begin sample construction") - const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); - const results = '${{ steps.validation.outputs.results }}'.split('\n\n'); + const samples = "${{ steps.sample_folders.outputs.samples }}".split('\n'); + const results = "${{ steps.validation.outputs.results }}".split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From 93ef96c11d248d2d6a2bf669ec6b3e34c5697e6a Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:01:54 -0500 Subject: [PATCH 60/75] Be rid of single quotes --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index e1653fd6..d70bc09f 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -51,8 +51,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | console.log("Begin sample construction") - const samples = "${{ steps.sample_folders.outputs.samples }}".split('\n'); - const results = "${{ steps.validation.outputs.results }}".split('\n\n'); + const samples = ${{ steps.sample_folders.outputs.samples }}.split('\n'); + const results = ${{ steps.validation.outputs.results }}.split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From a7ca23f2626fde184dce30893648bb5443384167 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:16:43 -0500 Subject: [PATCH 61/75] Try to institute and replace single quote escape around JSON rules --- .github/workflows/validate_samples.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index d70bc09f..3fc77969 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,6 +38,7 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true + result="${result//\'/\\\'}" results+="$result\n\n" done @@ -51,8 +52,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | console.log("Begin sample construction") - const samples = ${{ steps.sample_folders.outputs.samples }}.split('\n'); - const results = ${{ steps.validation.outputs.results }}.split('\n\n'); + const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); + const results = '${{ steps.validation.outputs.results }}'.split('\n\n'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; @@ -61,7 +62,7 @@ jobs: console.log(samples[i]) console.log(results[i]) const sample = samples[i]; - const result = JSON.parse(results[i]); + const result = JSON.parse(results[i].replace('\\\'','\'')); let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From d038ffcf25fdc5a1d6dcfde19c77c5ed8e1b7379 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:24:36 -0500 Subject: [PATCH 62/75] Get rid of single quotes at source --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 3fc77969..1b929943 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,7 +38,7 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true - result="${result//\'/\\\'}" + result="${result//\'/\\\"}" results+="$result\n\n" done @@ -62,7 +62,7 @@ jobs: console.log(samples[i]) console.log(results[i]) const sample = samples[i]; - const result = JSON.parse(results[i].replace('\\\'','\'')); + const result = JSON.parse(results[i]); let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From 3793dabb8d576afdec4af0b1e6e668062b07ac6e Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:38:19 -0500 Subject: [PATCH 63/75] Try parse step beforehand --- .github/workflows/validate_samples.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 1b929943..2ac73476 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -39,7 +39,7 @@ jobs: for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'/\\\"}" - results+="$result\n\n" + results+="$result,\n" done echo "RESULTS<> $GITHUB_OUTPUT @@ -53,7 +53,7 @@ jobs: script: | console.log("Begin sample construction") const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); - const results = '${{ steps.validation.outputs.results }}'.split('\n\n'); + const results = JSON.parse('[${{ steps.validation.outputs.results }}]'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; @@ -62,7 +62,7 @@ jobs: console.log(samples[i]) console.log(results[i]) const sample = samples[i]; - const result = JSON.parse(results[i]); + const result = results[i]; let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From 08f3ac41ac84ec373ceb486d7a9e1afa8ba39337 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:40:29 -0500 Subject: [PATCH 64/75] Remove invalid JSON escape --- .github/workflows/validate_samples.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 2ac73476..998157e7 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,7 +38,6 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true - result="${result//\'/\\\"}" results+="$result,\n" done From ccffa175e69b6033b5d7605fc5b9f58c8ba14776 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:55:10 -0500 Subject: [PATCH 65/75] Modify substitution --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 998157e7..d60d1aff 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,6 +38,7 @@ jobs: results="" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true + result="${result//\'s/}" results+="$result,\n" done From 693729ee351b54130570d4888f36cdeb9a2f34e8 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 15:59:34 -0500 Subject: [PATCH 66/75] Try to create valid JSON --- .github/workflows/validate_samples.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index d60d1aff..811b3d8c 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,13 +35,14 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - results="" + results="{"values": [" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true - result="${result//\'s/}" results+="$result,\n" done + results+="]}" + echo "RESULTS<> $GITHUB_OUTPUT echo ${results} >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT @@ -53,7 +54,7 @@ jobs: script: | console.log("Begin sample construction") const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); - const results = JSON.parse('[${{ steps.validation.outputs.results }}]'); + const results = JSON.parse(${{ steps.validation.outputs.results }}); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; @@ -62,7 +63,7 @@ jobs: console.log(samples[i]) console.log(results[i]) const sample = samples[i]; - const result = results[i]; + const result = results.values[i]; let status = ''; if (result.valid) { status = ':heavy_check_mark:'; From 10e659c253ff31e87e1a81835cdc8dfa63fce6e5 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:01:38 -0500 Subject: [PATCH 67/75] Different apostrophe handling --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 811b3d8c..ba9f9b40 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,6 +38,7 @@ jobs: results="{"values": [" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true + result="${result//\'s/}" results+="$result,\n" done @@ -54,7 +55,7 @@ jobs: script: | console.log("Begin sample construction") const samples = '${{ steps.sample_folders.outputs.samples }}'.split('\n'); - const results = JSON.parse(${{ steps.validation.outputs.results }}); + const results = JSON.parse('${{ steps.validation.outputs.results }}'); let comment = `**Data Validation Report**\n\n`; comment += '| Sample | Status |\n'; comment += '| ------ | ------ |\n'; From a7040376fc139683e12aee1283d27f7048a44dfd Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:04:10 -0500 Subject: [PATCH 68/75] Escape characters for values proclamation --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index ba9f9b40..568df197 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -35,7 +35,7 @@ jobs: id: validation run: | samples=(${{ steps.sample_folders.outputs.samples }}) - results="{"values": [" + results="{\"values\": [" for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" From aa083e116dd303d86db470165773874c57bd9d5b Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:06:51 -0500 Subject: [PATCH 69/75] Additional escape logic --- .github/workflows/validate_samples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 568df197..174f0e40 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -39,6 +39,7 @@ jobs: for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" + result="${result//\"/\\"}" results+="$result,\n" done From a0d9c8560dfb8bb146420327d760d8e9a0e88ddd Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:11:55 -0500 Subject: [PATCH 70/75] Better sequence match --- .github/workflows/validate_samples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 174f0e40..1119e27b 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -39,7 +39,7 @@ jobs: for sample in "${samples[@]}"; do result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" - result="${result//\"/\\"}" + result="${result//\\\"/\\\\\"}" results+="$result,\n" done From fbf483734cafebc2d03af9b42961926368d6bf19 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:19:15 -0500 Subject: [PATCH 71/75] Construct last array element properly --- .github/workflows/validate_samples.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 1119e27b..b1e6c558 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -36,13 +36,15 @@ jobs: run: | samples=(${{ steps.sample_folders.outputs.samples }}) results="{\"values\": [" + result="" for sample in "${samples[@]}"; do + results+="$result," result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" result="${result//\\\"/\\\\\"}" - results+="$result,\n" done + results+="$result" results+="]}" echo "RESULTS<> $GITHUB_OUTPUT From 95a369e28e16501b89ea132e8f26a622a51c29c0 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:21:22 -0500 Subject: [PATCH 72/75] Comma handling --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index b1e6c558..88456b59 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,10 +38,11 @@ jobs: results="{\"values\": [" result="" for sample in "${samples[@]}"; do - results+="$result," + results+="$result" result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" result="${result//\\\"/\\\\\"}" + result="$result," done results+="$result" From c48534feeefe24b735a7884149d7098c899870b6 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:27:06 -0500 Subject: [PATCH 73/75] Get rid of nagging comma --- .github/workflows/validate_samples.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 88456b59..9b5ed473 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,13 +38,13 @@ jobs: results="{\"values\": [" result="" for sample in "${samples[@]}"; do - results+="$result" + results+="$result," result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" result="${result//\\\"/\\\\\"}" - result="$result," done + result="${result%?}" results+="$result" results+="]}" From 09a70e54b4b6d7f8c605ef208b76b07805947f3d Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:28:41 -0500 Subject: [PATCH 74/75] Last comma change --- .github/workflows/validate_samples.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml index 9b5ed473..0e77c164 100644 --- a/.github/workflows/validate_samples.yml +++ b/.github/workflows/validate_samples.yml @@ -38,10 +38,11 @@ jobs: results="{\"values\": [" result="" for sample in "${samples[@]}"; do - results+="$result," + results+="$result" result=$(frictionless validate --schema-sync "${sample}/datapackage.json" --json) || true result="${result//\'s/}" result="${result//\\\"/\\\\\"}" + result="$result," done result="${result%?}" From 480640548c268c5cad717e1e1aec42460186469b Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Fri, 25 Aug 2023 16:38:24 -0500 Subject: [PATCH 75/75] Demonstrate passing validation --- samples/template/TIDES/datapackage.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json index e6aa189a..c667a37a 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -28,7 +28,7 @@ "name": "devices", "profile": "tabular-data-resource", "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/devices.schema.json", "sources": [ { "title": "Where did data come from?", @@ -42,7 +42,7 @@ "name": "vehicle_locations", "profile": "tabular-data-resource", "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_locations.schema.json", "sources": [ { "title": "Where did data come from?", @@ -56,7 +56,7 @@ "name": "fare_transactions", "profile": "tabular-data-resource", "path": "fare_transactions.csv", - "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "schema":"https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/fare_transactions.schema.json", "sources": [ { "title": "Where did data come from?", @@ -70,7 +70,7 @@ "name": "train_cars", "profile": "tabular-data-resource", "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -84,7 +84,7 @@ "name": "operators", "profile": "tabular-data-resource", "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/operators.schema.json", "sources": [ { "title": "Where did data come from?", @@ -98,7 +98,7 @@ "name": "stop_visits", "profile": "tabular-data-resource", "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/stop_visits.schema.json", "sources": [ { "title": "Where did data come from?", @@ -112,7 +112,7 @@ "name": "vehicle_train_cars", "profile": "tabular-data-resource", "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicle_train_cars.schema.json", "sources": [ { "title": "Where did data come from?", @@ -126,7 +126,7 @@ "name": "vehicles", "profile": "tabular-data-resource", "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/vehicles.schema.json", "sources": [ { "title": "Where did data come from?", @@ -140,7 +140,7 @@ "name": "trips_performed", "profile": "tabular-data-resource", "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/trips_performed.schema.json", "sources": [ { "title": "Where did data come from?", @@ -154,7 +154,7 @@ "name": "station_activities", "profile": "tabular-data-resource", "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/station_activities.schema.json", "sources": [ { "title": "Where did data come from?", @@ -168,7 +168,7 @@ "name": "passenger_events", "profile": "tabular-data-resource", "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/sample-data-doc/spec/passenger_events.schema.json", "sources": [ { "title": "Where did data come from?",