diff --git a/.github/workflows/validate_samples.yml b/.github/workflows/validate_samples.yml new file mode 100644 index 00000000..639d99e2 --- /dev/null +++ b/.github/workflows/validate_samples.yml @@ -0,0 +1,25 @@ +name: Validate-Samples + +on: + push: + paths: + - 'samples/*/TIDES/*' + - 'spec/*' + pull_request: + paths: + - 'samples/*/TIDES/*' + - 'spec/*' + workflow_dispatch: + create: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Validate sample data + uses: frictionlessdata/repository@v2 + with: + packages: "samples/*/TIDES/datapackage.json" + diff --git a/.gitignore b/.gitignore index 0c6439c5..64ff4eff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ .DS_Store* .vscode* +/__pycache__/* .env /venv* -/site -/__pycache__ +/site/* +# pages that are copied in from main repo +/docs/CONTRIBUTING.md +/docs/CODE_OF_CONDUCT.md +/docs/README.md +# pages that are generated from templates +/docs/tables.md +/docs/architecture.md diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 55f1ddb8..e150afdc 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -8,5 +8,11 @@ MD007: # Remove line length limit MD013: false +# Remove "fenced code blocks should have language specified" +MD040: false + +# Remove "no inline HTML" +MD033: false + # Remove "no emphasis as heading" MD036: false diff --git a/README.md b/README.md index 0998f081..8ba5a364 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,43 @@ This repository provides data schemas and tools to support the access, managemen ## TIDES specification -The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema standards. +The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema and [data package](https://specs.frictionlessdata.io/data-package) standards. Human-friendlier documentation is auto-generated and available at: - [Architecture](architecture.md) - [Table Schemas](tables.md) +## Example Data + +Sample data can be found in the `/samples` directory, with one directory for each example. + +## Validating TIDES data + +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Data Package + +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/datapackage.json). + +Once this is created, mapping the data files to the schema, simply run: + +```sh +frictionless validate datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +``` + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in . diff --git a/contributors.md b/contributors.md index 405813b6..77e81149 100644 --- a/contributors.md +++ b/contributors.md @@ -1 +1 @@ -# Contributors \ No newline at end of file +# Contributors diff --git a/docs/index.md b/docs/index.md index e31d914c..2578b68e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1 +1,3 @@ -{{ include_file('README.md', downshift_h1= False) }} +# TIDES Transit Specification Suite + +{{ include_file('README.md', start_line = 2, downshift_h1= False) }} diff --git a/docs/samples.md b/docs/samples.md new file mode 100644 index 00000000..7b4125ca --- /dev/null +++ b/docs/samples.md @@ -0,0 +1,9 @@ +# Sample Data + +Sample data can be found in the `/samples` directory, with one directory for each data sample. + +{{ include_file('samples/README.md')}} + +## Data List + +{{ list_samples('samples') }} diff --git a/main.py b/main.py index 88d2e76d..cf106afb 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ "tables.md": "tables", } + def define_env(env): """ This is the hook for defining variables, macros and filters @@ -26,7 +27,33 @@ def define_env(env): """ @env.macro - def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_line: int = None): + def list_samples(data_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + Args: + data_dir (str):directory to search in + Returns: + str: markdown-formatted list + """ + + fields = ["Sample", "Agency", "Resources", "Vendors"] + + data_dir = os.path.join(env.project_dir, data_dir) + samples = [ + d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) + ] + + md_table = ( + "| *" + "** | **".join(fields) + "* |\n| " + " ----- |" * len(fields) + "\n" + ) + + for s in samples: + md_table += datapackage_to_row(os.path.join(data_dir, s), fields) + return md_table + + @env.macro + def include_file( + filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None + ): """ Include a file, optionally indicating start_line and end_line. @@ -34,7 +61,8 @@ def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_li args: filename: file to include, relative to the top directory of the documentation project. - downshift_h1: If true, will downshift headings by 1 if h1 heading found. Defaults to True. + downshift_h1: If true, will downshift headings by 1 if h1 heading found. + Defaults to True. start_line (Optional): if included, will start including the file from this line (indexed to 0) end_line (Optional): if included, will stop including at this line (indexed to 0) @@ -53,20 +81,19 @@ def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_li 4: re.compile(r"(#{4}\s)(.*)"), 5: re.compile(r"(#{5}\s)(.*)"), } - print(f"???before downshifting! {full_filename}") + if md_heading_re[1].search(content) and downshift_h1: - print("!!!downshifting!") - content = re.sub(md_heading_re[5],r'#\1\2',content) - content = re.sub(md_heading_re[4],r'#\1\2',content) - content = re.sub(md_heading_re[3],r'#\1\2',content) - content = re.sub(md_heading_re[2],r'#\1\2',content) - content = re.sub(md_heading_re[1],r'#\1\2',content) + content = re.sub(md_heading_re[5], r"#\1\2", content) + content = re.sub(md_heading_re[4], r"#\1\2", content) + content = re.sub(md_heading_re[3], r"#\1\2", content) + content = re.sub(md_heading_re[2], r"#\1\2", content) + content = re.sub(md_heading_re[1], r"#\1\2", content) _filenamebase = env.page.file.url for _find, _replace in FIND_REPLACE.items(): if _filenamebase in _replace: _replace = _replace.replace(_filenamebase, "") - + content = content.replace(_find, _replace) return content @@ -95,7 +122,7 @@ def frictionless_spec( columns=["fullpath", "fullpath_schema", "path", "schema", "name"] ).reset_index() spec_df["name"] = spec_df["name"].apply( - lambda x: f"[`{x}`](tables.md#{x})".replace("_","-") + lambda x: f"[`{x}`](tables.md#{x})".replace("_", "-") ) return spec_df.to_markdown(index=False) @@ -133,6 +160,49 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] +def datapackage_to_row(dir: str, fields: list) -> str: + """Reads datapackage and translates key metadata to a row in a markdown table. + + Right now, only works for fields: "Sample", "Agency", "Resources", "Vendors" + + Args: + dir (str): fully qualified directory for sample data + fields (list): list of fields to return + + Returns: + str: row in a markdown table with name, agency, resources, vendors + """ + dp_filename = os.path.join(dir, "datapackage.json") + if not os.path.isfile(dp_filename): + raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + _vendors = [ + s.get("vendor", None) for r in dp["resources"] for s in r.get("sources", []) + ] + _vendors = list(set(_vendors) - set([None])) + + _md_cells = { + "Sample": _to_sample_readme_link(dp["name"], dir), + "Agency": dp["agency"], + "Resources": ( + "" + ), + "Vendors": "", + } + + md_row = "| " + " | ".join([_md_cells[f] for f in fields]) + " |\n" + + return md_row + + +def _to_sample_readme_link(sample_name, folder_dir): + return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" + + def _document_frictionless_schema(schema_filename: str) -> dict: """Documents a single frictionless schema. diff --git a/mkdocs.yml b/mkdocs.yml index b88dbedb..7519dc10 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,8 +34,19 @@ plugins: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' + - mike: + version_selector: true + - redirects: + redirect_maps: #original relative to /docs : redirect target + '../CODE_OF_CONDUCT.md': 'CODE_OF_CONDUCT.md' + '../CONTRIBUTING.md': 'CONTRIBUTING.md' + '../contributors.md': 'contributors.md' - search +extra: + version: + provider: mike + watch: - CONTRIBUTING.md - CODE_OF_CONDUCT.md @@ -72,4 +83,5 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md + - Sample Data: samples.md - Development: development.md diff --git a/requirements.txt b/requirements.txt index d8dec895..b09f5df1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ +-r docs/requirements.txt pre-commit --r docs/requirements.txt \ No newline at end of file diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..a0307793 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,76 @@ +# Data Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +``` +unique-example-name + \TIDES # Required. Data to be validated against the TIDES specification + \datapackage.json # Required. Data package metadata per https://specs.frictionlessdata.io/data-package/ + \raw # Optional. Data which the agency uses to create TIDES data + \scripts # Optional. Scripts used to transform raw --> TIDES +``` + +## Adding Examples + +We encourage the addition of examples, but please follow the following guidelines: + +1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. All individual files should be well under 50 MB. +2. *Include Metadata* as specified in [`datapackage.json`](#data-package). +3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. + +## Data Package + +TIDES data packages must include a [`datapackage.json`](https://specs.frictionlessdata.io/data-package/). Key information to include in [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) includes: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | A human-readable title. | Required | +| `name` | Identifier string as a URL-friendly slug. | Required | +| `description` | Short description of data package. | Recommended | +| `agency` | Transit agency name. | Recommended | +| `ntd_id` | ID for the National Transit Database. | Recommended | +| `profile` | Should be `tabular-data-package` | Required | +| `licenses` | Should be `[{"name": "Apache-2.0"}]` to be consistent with this repository | Required | +| `contributors` | Array of data contributors `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `maintainers` | Array of data maintainers `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `resources` | Array of data files included in your package, formated as a [`tabular-data-resource`](#data-resource)| Required | + +### Data Resource + +Key fields for each [`tabular-data-resource`](https://specs.frictionlessdata.io/tabular-data-resource/) are as follows: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `name` | Short sluggable name used to refer to data in this file. | Required | +| `path` | Path of the data resource file relative to the `datapackage.json` | Required | +| `schema` | Data schema to use to valdiate the data resource to | Required | +| `sources` | Array of data sources formatted as a [`source`](#data-source) | Recommended | + +### Data Source + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | Description of the data source. | Required | +| `component` | What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc. | Recommended | +| `product` | What product was used to generate this data (directly or indirectly)? | Recommended | +| `vendor` | What company makes this product? | Recommended | + +## Data validation + +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json +``` +### Continuous Data Validation + +Example data in the `\TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch.