diff --git a/.gitignore b/.gitignore index 0c6439c5..64ff4eff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ .DS_Store* .vscode* +/__pycache__/* .env /venv* -/site -/__pycache__ +/site/* +# pages that are copied in from main repo +/docs/CONTRIBUTING.md +/docs/CODE_OF_CONDUCT.md +/docs/README.md +# pages that are generated from templates +/docs/tables.md +/docs/architecture.md diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 71f3b6e9..90f5fe5c 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -11,7 +11,10 @@ MD041: false # Remove line length limit MD013: false -# Remove code block style because it also fails on admonition +# Remove "fenced code blocks should have language specified" +MD040: false + +# Remove "no inline HTML" MD046: false # Remove no in line HTML diff --git a/README.md b/README.md index 62de1b4b..ef3934a7 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,43 @@ This repository provides data schemas and tools to support the access, managemen ## TIDES specification -The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema standards. +The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema and [data package](https://specs.frictionlessdata.io/data-package) standards. Human-friendlier documentation is auto-generated and available at: - [Architecture](https://tides-transit.github.io/TIDES/main/architecture) - [Table Schemas](https://tides-transit.github.io/TIDES/main/tables) +## Example Data + +Sample data can be found in the `/samples` directory, with one directory for each example. + +## Validating TIDES data + +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Data Package + +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/datapackage.json). + +Once this is created, mapping the data files to the schema, simply run: + +```sh +frictionless validate datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json --schema-sync +``` + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/docs/index.md b/docs/index.md index e31d914c..2578b68e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1 +1,3 @@ -{{ include_file('README.md', downshift_h1= False) }} +# TIDES Transit Specification Suite + +{{ include_file('README.md', start_line = 2, downshift_h1= False) }} diff --git a/docs/samples.md b/docs/samples.md new file mode 100644 index 00000000..7b4125ca --- /dev/null +++ b/docs/samples.md @@ -0,0 +1,9 @@ +# Sample Data + +Sample data can be found in the `/samples` directory, with one directory for each data sample. + +{{ include_file('samples/README.md')}} + +## Data List + +{{ list_samples('samples') }} diff --git a/main.py b/main.py index 820c0711..095780ab 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,30 @@ def define_env(env): - macro: a decorator function, to declare a macro. """ + @env.macro + def list_samples(data_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + Args: + data_dir (str):directory to search in + Returns: + str: markdown-formatted list + """ + + fields = ["Sample", "Agency", "Resources", "Vendors"] + + data_dir = os.path.join(env.project_dir, data_dir) + samples = [ + d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) + ] + + md_table = ( + "| *" + "** | **".join(fields) + "* |\n| " + " ----- |" * len(fields) + "\n" + ) + + for s in samples: + md_table += datapackage_to_row(os.path.join(data_dir, s), fields) + return md_table + @env.macro def include_file( filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None @@ -57,9 +81,8 @@ def include_file( 4: re.compile(r"(#{4}\s)(.*)"), 5: re.compile(r"(#{5}\s)(.*)"), } - print(f"???before downshifting! {full_filename}") + if md_heading_re[1].search(content) and downshift_h1: - print("!!!downshifting!") content = re.sub(md_heading_re[5], r"#\1\2", content) content = re.sub(md_heading_re[4], r"#\1\2", content) content = re.sub(md_heading_re[3], r"#\1\2", content) @@ -137,6 +160,51 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] +def datapackage_to_row(dir: str, fields: list) -> str: + """Reads datapackage and translates key metadata to a row in a markdown table. + + Right now, only works for fields: "Sample", "Agency", "Resources", "Vendors" + + Args: + dir (str): fully qualified directory for sample data + fields (list): list of fields to return + + Returns: + str: row in a markdown table with name, agency, resources, vendors + """ + dp_filename = os.path.join(dir, "datapackage.json") + if not os.path.isfile(dp_filename): + raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + _vendors = [ + s.get("vendor", None) for r in dp["resources"] for s in r.get("sources", []) + ] + _vendors += dp.get("sources",{}).get("vendor",[]) + + _vendors = list(set(_vendors) - set([None])) + + _md_cells = { + "Sample": _to_sample_readme_link(dp["name"], dir), + "Agency": dp["agency"], + "Resources": ( + "" + ), + "Vendors": "", + } + + md_row = "| " + " | ".join([_md_cells[f] for f in fields]) + " |\n" + + return md_row + + +def _to_sample_readme_link(sample_name, folder_dir): + return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" + + def _document_frictionless_schema(schema_filename: str) -> dict: """Documents a single frictionless schema. diff --git a/mkdocs.yml b/mkdocs.yml index 8cddcdc8..4dda7826 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,8 +34,19 @@ plugins: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' + - mike: + version_selector: true + - redirects: + redirect_maps: #original relative to /docs : redirect target + '../CODE_OF_CONDUCT.md': 'CODE_OF_CONDUCT.md' + '../CONTRIBUTING.md': 'CONTRIBUTING.md' + '../contributors.md': 'contributors.md' - search +extra: + version: + provider: mike + watch: - CONTRIBUTING.md - CODE_OF_CONDUCT.md @@ -72,4 +83,5 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md + - Sample Data: samples.md - Development: development.md diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..8c34b6a8 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,75 @@ +# Data Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +``` +unique-example-name + \TIDES # Required. Data to be validated against the TIDES specification + \datapackage.json # Required. Data package metadata per https://specs.frictionlessdata.io/data-package/ + \raw # Optional. Data which the agency uses to create TIDES data + \scripts # Optional. Scripts used to transform raw --> TIDES +``` + +## Adding Examples + +We encourage the addition of examples, but please follow the following guidelines: + +1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. The whole data package should be well under 50 MB. +2. *Include Metadata* as specified in [`datapackage.json`](#data-package). +3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. + +## Data Package + +TIDES data packages must include a [`datapackage.json`](https://specs.frictionlessdata.io/data-package/). Key information to include in [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) includes: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | A human-readable title. | Required | +| `name` | Short [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string. | Recommended | +| `description` | Short description of data package. | Recommended | +| `agency` | Transit agency name. | Recommended | +| `ntd_id` | ID for the National Transit Database. | Recommended | +| `profile` | Should be `tabular-data-package` | Required | +| `licenses` | Should be `[{"name": "Apache-2.0"}]` to be consistent with this repository | Required | +| `contributors` | Array of data contributors `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `maintainers` | Array of data maintainers `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `sources` | Array of data sources formatted as a [`source`](#data-source). Recommended to be documented *either* here at the top-level or for each individual `resource`. | Recommended | +| `resources` | Array of data files included in your package, formated as a [`tabular-data-resource`](#data-resource)| Required | + +### Data Resource + +Key fields for each [`tabular-data-resource`](https://specs.frictionlessdata.io/tabular-data-resource/) are as follows: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `name` | Short [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string used to refer to data in this file. `name` must be unique within this datapackage. | Required | +| `path` | Path of the data resource file relative to the `datapackage.json` | Required | +| `schema` | Data schema to use to valdiate the data resource to | Required | +| `sources` | Array of data sources formatted as a [`source`](#data-source). Recommended to be documented *either* here for each individual `resource` or at the top-level. | Recommended | + +### Data Source + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | Description of the data source. | Required | +| `component` | What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc. | Recommended | +| `product` | What product was used to generate this data (directly or indirectly)? | Recommended | +| `product_version` | Describe the version of the product was used. | Optional | +| `vendor` | What company makes this product? | Recommended | + +## Data validation + +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json --schema-sync +```