From a24f119400d0e4b18754713ab4790658690c4ac1 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 26 Sep 2022 15:41:06 -0700 Subject: [PATCH 01/21] Merge content from main repo using Macro include_files Use this instead because it is more configurable/extensible than the redirects. --- .gitignore | 3 ++- contributors.md | 2 +- docs/contributors.md | 1 - docs/development.md | 7 +++++++ docs/document.py | 4 ---- docs/index.md | 1 + 6 files changed, 11 insertions(+), 7 deletions(-) delete mode 100644 docs/contributors.md create mode 100644 docs/development.md create mode 100644 docs/index.md diff --git a/.gitignore b/.gitignore index 8b09edef..64ff4eff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ .DS_Store* .vscode* +/__pycache__/* .env /venv* -/site +/site/* # pages that are copied in from main repo /docs/CONTRIBUTING.md /docs/CODE_OF_CONDUCT.md diff --git a/contributors.md b/contributors.md index 90d03755..da2ae884 100644 --- a/contributors.md +++ b/contributors.md @@ -1 +1 @@ -# Contributors to the TIDES Suite \ No newline at end of file +# Contributors to the TIDES Suite diff --git a/docs/contributors.md b/docs/contributors.md deleted file mode 100644 index da2ae884..00000000 --- a/docs/contributors.md +++ /dev/null @@ -1 +0,0 @@ -# Contributors to the TIDES Suite diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..7b436a3d --- /dev/null +++ b/docs/development.md @@ -0,0 +1,7 @@ +# Development + +{{ include_file('CONTRIBUTING.md') }} + +{{ include_file('CODE_OF_CONDUCT.md') }} + +{{ include_file('contributors.md') }} diff --git a/docs/document.py b/docs/document.py index 1965f703..9153e19a 100644 --- a/docs/document.py +++ b/docs/document.py @@ -299,7 +299,3 @@ def repo_to_docs( if __name__ == "__main__": document_spec() document_schemas() - repo_to_docs("README.md") - repo_to_docs("CONTRIBUTING.md") - repo_to_docs("contributors.md") - repo_to_docs("CODE_OF_CONDUCT.md") diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..81b5d74d --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +{{ include_file('README.md') }} From 74b02d9b07261f9997071c14ee595d7774284327 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 26 Sep 2022 15:42:01 -0700 Subject: [PATCH 02/21] [ actually include the macros ] --- docs/examples.md | 24 ++++++++++++++++++++++++ main.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 71 insertions(+) create mode 100644 docs/examples.md create mode 100644 main.py diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 00000000..66af3883 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,24 @@ +# Examples + +[![Example Data](https://github.com/TIDES-transit/TIDES/actions/workflows/validate-data.yaml/badge.svg)](https://repository.frictionlessdata.io/pages/dashboard.html?user=TIDES-transit&repo=TIDES&flow=validate-data) + +Example data can be found in the `/data` directory, with one directory for each example. + +## Example validation + +Example data in the `\TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema contained in the respective repository commit. + +## Example Directory Organization + +Each example should follow the following directory structure: + +``` +unique-example-name + \TIDES # data to be validated against the TIDES specification + \raw # data which the agency uses to create TIDES data + \scripts # scripts used to transform raw --> TIDES +``` + +## Example List + +{{ testingme }} diff --git a/main.py b/main.py new file mode 100644 index 00000000..6b452a2b --- /dev/null +++ b/main.py @@ -0,0 +1,46 @@ +import os +import pathlib + +FIND_REPLACE = { # original relative to /docs : redirect target + "CODE_OF_CONDUCT.md": "development/#CODE_OF_CONDUCT", + "CONTRIBUTING.md": "development/#CONTRIBUTING", + "contributors.md": "development/#contributors", + "architecture.md": "architecture", + "tables.md": "tables", +} + + +def define_env(env): + """ + This is the hook for defining variables, macros and filters + + - variables: the dictionary that contains the environment variables + - macro: a decorator function, to declare a macro. + """ + + @env.macro + def include_file(filename: str, start_line: int = 0, end_line: int = None): + """ + Include a file, optionally indicating start_line and end_line. + + Will create redirects if specified in FIND_REPLACE in main.py. + + args: + filename: file to include, relative to the top directory of the documentation project. + start_line (Optional): if included, will start including the file from this line + (indexed to 0) + end_line (Optional): if included, will stop including at this line (indexed to 0) + """ + full_filename = os.path.join(env.project_dir, filename) + with open(full_filename, "r") as f: + lines = f.readlines() + line_range = lines[start_line:end_line] + content = "".join(line_range) + + _filenamebase = env.page.file.url + + for _find, _replace in FIND_REPLACE.items(): + if _filenamebase in _replace: + _replace = _replace.replace(_filenamebase, "") + content = content.replace(_find, _replace) + return content diff --git a/mkdocs.yml b/mkdocs.yml index 1a808f67..0ad62f08 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,6 +26,7 @@ theme: plugins: - autorefs - awesome-pages + - macros: - mermaid2: arguments: theme: | From 9db30a970130d018cbae61c4ca5de576ad70f440 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 28 Sep 2022 09:10:22 -0700 Subject: [PATCH 03/21] Add Examples --- .github/workflows/validate-data.yml | 22 ++ README.md | 8 + docs/requirements.txt | 3 +- main.py | 322 ++++++++++++++++++++++++++++ mkdocs.yml | 15 +- 5 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/validate-data.yml diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml new file mode 100644 index 00000000..dbe7db4a --- /dev/null +++ b/.github/workflows/validate-data.yml @@ -0,0 +1,22 @@ +name: Validate Example TIDES Data + +on: + push: + paths: + - 'data/*/TIDES/*' + pull_request: + paths: + - 'data/*/TIDES/*' + workflow_dispatch: + create: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Validate data + uses: frictionlessdata/repository@v2 + with: + packages: "data/*/TIDES/*.package.y?ml" diff --git a/README.md b/README.md index 22029038..767ed742 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,14 @@ Human-friendlier documentation is auto-generated and available at: - [Architecture](architecture.md) - [Table Schemas](tables.md) +## Example Data + +[![Example Data](https://github.com/TIDES-transit/TIDES/actions/workflows/validate-data.yaml/badge.svg)](https://repository.frictionlessdata.io/pages/dashboard.html?user=TIDES-transit&repo=TIDES&flow=validate-data) + +Example data can be found in the `/data` directory, with one directory for each example. + +Example data in the `/TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema contained in the respective repository commit. + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in the [CONTRIBUTING.md](CONTRIBUTING.md) file. diff --git a/docs/requirements.txt b/docs/requirements.txt index 561624f8..e3c5b666 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,5 +6,6 @@ mkdocs-jupyter mkdocs-macros-plugin mkdocs-material mkdocs-mermaid2-plugin +mkdocs-redirects pandas -tabulate \ No newline at end of file +tabulate diff --git a/main.py b/main.py index 6b452a2b..3799597d 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,11 @@ +import glob +import json import os import pathlib +import re +from typing import Union + +import pandas as pd FIND_REPLACE = { # original relative to /docs : redirect target "CODE_OF_CONDUCT.md": "development/#CODE_OF_CONDUCT", @@ -44,3 +50,319 @@ def include_file(filename: str, start_line: int = 0, end_line: int = None): _replace = _replace.replace(_filenamebase, "") content = content.replace(_find, _replace) return content + + @env.macro + def frictionless_spec(spec_filename:str= None) -> str: + if not base_path: + base_path = os.path.dirname(os.path.realpath(__file__)) + if not docs_path: + docs_path = os.path.join(base_path, "docs") + + if spec_filename is None: + spec_filename = glob.glob(os.path.join(base_path, "**/*.spec.json"), recursive=True)[0] + + print("Documenting spec_file: ", spec_filename) + + # Generate a table for overall file requirements + spec_df = read_config(spec_filename) + spec_df = spec_df.drop( + columns=["fullpath", "fullpath_schema", "path", "schema", "name"] + ).reset_index() + spec_df["name"] = spec_df["name"].apply( + lambda x: "[`{}`](tables.md#{})".format(x, x) + ) + + _fill_template( + outfile_path=os.path.join(docs_path, outfile_name), + content_dict={"SPEC": spec_df.to_markdown(index=False)}, + ) + + +def _format_cell_by_type(x, header): + if "enum" in header and len(x) > 1: + return "Allowed Values: `" + "`,`".join(map(str, x)) + "`" + if header == "foreign_key" and len(x) > 1: + table, var = x.split(".") + if table == "": + return "Variable: `{}`".format(var) + return "Table: `{}`, Variable: `{}`".format(table, var) + else: + return str(x).replace("\n", "
") + + +def _recursive_items(d: dict, key_prefix: str = ""): + """ + Recursively flattens a nested dictionary and returns a dictionary with key_prefixe + to represent original nesting structures. + Modified from https://stackoverflow.com/questions/39233973/get-all-keys-of-a-nested-dictionary + """ + for k, v in d.items(): + if k == "constraints": + _str = "" + if v.pop("required", None): + _str += "**Required**\n" + if v.pop("unique", None): + _str += "*Unique*\n" + enum = [str(x) for x in v.pop("enum", [])] + if enum: + _str += "Allowed Values\n - `" + "`\n- `".join(enum) + "`" + _remove = ["{", "}", "[", "]", '"', ","] + _json = json.dumps(v, indent=2) + _str += "".join(s for s in _json if s not in _remove) + yield (k, _str) + + elif type(v) is dict: + yield from _recursive_items(v, key_prefix=k + " ") + else: + yield (key_prefix + k, v) + + +def _fill_template( + outfile_path: Union[str, pathlib.Path], + content_dict: dict, + template_path: Union[str, pathlib.Path] = None, +) -> None: + """Replace contents of a text file with template variables filled by a dictionary. + + Args: + outfile_path (Union[str,pathlib.Path]): File to write output to. + content_dict (dict[): Dictionary of values to fill template with. + template_path (Union[str,pathlib.Path], optional): _description_. If not specified, will assume + it is the outfile with a .template suffix (i.e. index.template.md) + + Raises: + ValueError: _description_ + """ + + outfile_path = pathlib.Path(outfile_path) + if template_path is None: + template_path = outfile_path.parent / pathlib.Path( + outfile_path.stem + ".template" + outfile_path.suffix + ) + + with open(template_path) as _template_file: + _txt_contents = _template_file.read().format(**content_dict) + with open(os.path.join(outfile_path), "w") as _outfile: + _outfile.write(_txt_contents) + + +def _list_to_md_table(list_of_dicts: list) -> str: + """ + Reads a list of dictionaries defining a schema and creates a markdown table. + args: + list_of_dicts: a list of dictionaries containing definition of fields. + returns: A markdown string representing the table. + """ + + # flatten dictionary + flat_list = [] + for i in list_of_dicts: + flat_list.append({k: v for k, v in _recursive_items(i)}) + + # get all items listed + standard_header_fields = [ + "name", + "constraints", + "type", + # "foreign_key", + "description", + ] + + _all_header_fields = [k for i in flat_list for k, v in i.items()] + _additional_header_fields = list( + set(_all_header_fields) - set(standard_header_fields) + ) + _header_fields = standard_header_fields + sorted(_additional_header_fields) + + header_md = "|" + "|".join(_header_fields) + "|\n" + header_md += "|---" * len(_header_fields) + "|\n" + + body_md = "" + for d in flat_list: + body_md += ( + "|" + + "|".join([_format_cell_by_type(d.get(i, " "), i) for i in _header_fields]) + + "|\n" + ) + + return header_md + body_md + + +def read_schema(schema_file: str) -> dict: + """ + Reads in schema from schema json file and returns as dictionary. + + Args: + schema_file: File location of the schema json file. + Returns: The schema as a dictionary + """ + with open(schema_file, encoding="utf-8") as f: + schema = json.load(f) + return schema + + +def read_config( + config_path: Union[pathlib.Path, str], + data_dir: Union[pathlib.Path, str] = None, + schema_dir: Union[pathlib.Path, str] = None, +) -> pd.DataFrame: + """ + Reads a frictionliess spec config file, adds some full paths and returns as a dataframe. + + Args: + config_path: Configuration file. A json file with a list of "resources" + specifying the "name", "path", and "schema" for each spec table as + well as a boolean value for "required". + Example: + :: + { + "resources": [ + { + "name":"link", + "path": "link.csv", + "schema": "link.schema.json", + "required": true + }, + { + "name":"node", + "path": "node.csv", + "schema": "node.schema.json", + "required": true + } + } + data_dir: Directory where example files go. + schema_dir: Directory where schema files are. If not specified, assumes + the same directory as the config_file. + Returns: frictionless configuration file as a DataFrame. + """ + config_path = pathlib.Path(config_path) + if data_dir is None: + data_dir = config_path.parent + if schema_dir is None: + schema_dir = config_path.parent + + with open(config_path, encoding="utf-8") as config_file: + config = json.load(config_file) + + resource_df = pd.DataFrame(config["resources"]) + + resource_df["required"].fillna(False, inplace=True) + + # Add full paths + resource_df["fullpath"] = resource_df["path"].apply( + lambda x: os.path.join(data_dir, x) + ) + resource_df["fullpath_schema"] = resource_df["schema"].apply( + lambda x: os.path.join(schema_dir, x) + ) + + resource_df.set_index("name", drop=False, inplace=True) + return resource_df + + +def document_schemas( + base_path: str = "", + docs_path: str = "", + outfile_name="tables.md", +) -> None: + """Document frictionless table schema files as markdown tables. + + Args: + base_path (str, optional): base path of repo. Defaults to two directories up from this file. + docs_path (str, optional): path where documentation is written. Defaults to "docs" within + the base_path. + outfile_name (str,optional): name of the file (and corresponding template) to write to. + Defaults to architecture.md. + out_path (str, optional): _description_. Defaults to ''. + """ + + if not base_path: + base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if not docs_path: + docs_path = os.path.join(base_path, "docs") + + # Create markdown with a table for each schema file + + schema_files = glob.glob( + os.path.join(base_path, "**/*.schema.json"), recursive=True + ) + print("Documenting schemas from: {}".format(schema_files)) + + file_schema_markdown = "" + for s in schema_files: + print(f"Documenting Schema: {s}") + spec_name = s.split("/")[-1].split(".")[0] + schema = read_schema(s) + file_schema_markdown += "\n\n## {}\n".format(spec_name) + file_schema_markdown += "\n\n{}".format(_list_to_md_table(schema["fields"])) + + _fill_template( + outfile_path=os.path.join(docs_path, outfile_name), + content_dict={"TABLES": file_schema_markdown}, + ) + + +def document_spec( + base_path: str = "", + docs_path: str = "", + outfile_name="architecture.md", +) -> None: + """Translate the frictionless .spec file to a markdown table. + + Args: + base_path (str, optional): base path of repo. Defaults to two directories up from this file. + docs_path (str, optional): path where documentation is written. Defaults to "docs" within + the base_path. + outfile_name (str,optional): name of the file (and corresponding template) to write to. + Defaults to architecture.md. + """ + if not base_path: + base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if not docs_path: + docs_path = os.path.join(base_path, "docs") + + spec_file = glob.glob(os.path.join(base_path, "**/*.spec.json"), recursive=True)[0] + + print("Documenting spec_file: ", spec_file) + + # Generate a table for overall file requirements + spec_df = read_config(spec_file) + spec_df = spec_df.drop( + columns=["fullpath", "fullpath_schema", "path", "schema", "name"] + ).reset_index() + spec_df["name"] = spec_df["name"].apply( + lambda x: "[`{}`](tables.md#{})".format(x, x) + ) + + _fill_template( + outfile_path=os.path.join(docs_path, outfile_name), + content_dict={"SPEC": spec_df.to_markdown(index=False)}, + ) + + +def repo_to_docs( + infile_name, outfile_name: str = None, base_path: str = "", docs_path: str = "" +): + """Copy files from repo to docs (i.e. README). + + Args: + infile_name (str): file to transfer from base repo to documentation directory + outfile_name (str): what to call the outfile in docs dir. Defaults to infile_name. + base_path (str, optional): _description_. Defaults to ''. + docs_path (str, optional): _description_. Defaults to ''. + """ + if not outfile_name: + outfile_name = infile_name + if not base_path: + base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if not docs_path: + docs_path = os.path.join(base_path, "docs") + + with open(os.path.join(base_path, infile_name), "rt") as fin: + with open(os.path.join(docs_path, outfile_name), "wt") as fout: + for line in fin: + outline = line + # add any fixes that need to happen here. + fout.write(outline) + + diff --git a/mkdocs.yml b/mkdocs.yml index 0ad62f08..92b0dbec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,13 +27,21 @@ plugins: - autorefs - awesome-pages - macros: + include_yaml: + - data/examples.yml - mermaid2: arguments: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' + - mike: + version_selector: true - search +extra: + version: + provider: mike + extra_javascript: - https://unpkg.com/mermaid/dist/mermaid.min.js @@ -61,9 +69,8 @@ markdown_extensions: permalink: " ΒΆ" nav: - - Home: README.md + - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md - - Contributing: CONTRIBUTING.md - - Contributors: contributors.md - - Code of Conduct: CODE_OF_CONDUCT.md \ No newline at end of file + - Contributing: development.md + - Example Data: examples.md From 7fdddfedadfdf91b339c39d7a2312b6ac178d1fe Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 28 Sep 2022 09:11:11 -0700 Subject: [PATCH 04/21] first hack at creating random example data --- data/my-agency-name/scripts/create_example.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 data/my-agency-name/scripts/create_example.py diff --git a/data/my-agency-name/scripts/create_example.py b/data/my-agency-name/scripts/create_example.py new file mode 100644 index 00000000..afab63a9 --- /dev/null +++ b/data/my-agency-name/scripts/create_example.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import glob +import json +import os +import pathlib +from random import choice, randrange +from datetime import timedelta + +import numpy as np +import pandas as pd + +EXAMPLE_DIR = os.path.dirname(os.path.abspath(__file__)) +BASE_REPO_DIR = os.path.dirname(os.path.dirname(os.path.dirname(EXAMPLE_DIR))) +TIDES_SPEC = os.path.join(BASE_REPO_DIR, "spec") + +SCHEMAS = glob.glob(os.path.join(TIDES_SPEC, "**/*.schema.json"), recursive=True) + +frictionless_to_pandas_types = { + "date": "datetime64[ns]", + "datetime": "datetime64[ns]", + "time": "datetime64[ns]", + "string": "object", + "integer": "int", + "float": "float64", + "number": "float64", + "boolean": "bool", +} + + +def write_schema_examples( + out_dir: str, + schemas: list = SCHEMAS, + generate_random: bool = False, + example_length: int = 5, +) -> None: + for s in schemas: + write_csv_for_schema( + s, out_dir, generate_random=generate_random, example_length=example_length + ) + + +def write_csv_for_schema( + schema_filename: str, + out_dir: str, + generate_random: bool = False, + example_length: int = 5, +) -> None: + """Creates blank or randomly filled csvs which comply with a schema. + + Args: + schema_filename (str): Filename with the Frictionless data schema + out_dir (str): Where the csv will be written + generate_random (bool): If true, will generate Defaults to False. + example_length (int): If generate_random is True, will generate this number of records. Defaults to 5. + """ + schema = read_schema(schema_filename) + schema_name = pathlib.Path(schema_filename).stem.split(".")[0] + fields_types = { + f["name"]: frictionless_to_pandas_types[f["type"]] for f in schema["fields"] + } + fields_series = {f: pd.Series(dtype=t) for f, t in fields_types.items()} + df = pd.DataFrame(fields_types) + out_filename = os.path.join(out_dir, schema_name + ".csv") + df.to_csv(out_filename, index=False) + + +def field_to_pd_series(field_def: dict, length: int): + default_min = 0 + default_max = 100 + if field_def["type"] in ["number","integer"]: + _min = field_def.get("constraints", {}).get("minimum", default_min) + _max = field_def.get("constraints", {}).get("maximum", default_min) + return pd.Series( + np.random.randint(_min, _max, length), + dtype=frictionless_to_pandas_types[field_def["type"]], + ) + elif field_def["type"] == "float": + _min = field_def.get("constraints", {}).get("minimum", default_min) + _max = field_def.get("constraints", {}).get("maximum", default_min) + return pd.Series( + np.random.rand(_min, _max) * length, + dtype=frictionless_to_pandas_types[field_def["type"]], + ) + elif field_def["type"] == "string": + default_str = "a value" + _enum = field_def.get("constraints", {}).get("enum", [default_str]) + return pd.Series( + [choice(_enum) for x in range(length)], + dtype=frictionless_to_pandas_types[field_def["type"]], + ) + elif field_def["type"]=="date": + + + elif field_def["type"]=="time": + + elif field_def["type"]=="datetime": + + else: + raise ValueError(f'Don't recognize {field_def["type"]') + + +def read_schema(schema_file: str) -> dict: + """ + Reads in schema from schema json file and returns as dictionary. + + Args: + schema_file: File location of the schema json file. + Returns: The schema as a dictionary + """ + with open(schema_file, encoding="utf-8") as f: + schema = json.load(f) + return schema + +default_start_datetime = datetime.strptime('1/1/2022 1:30 PM', '%m/%d/%Y %I:%M %p') +default_end_datetime = datetime.strptime('1/9/2022 6:30 PM', '%m/%d/%Y %I:%M %p' + +def random_date(start: datetime.datetime = , end: datetime.datetime): + """ + This function will return a random datetime between two datetime + objects. + + Source: + https://stackoverflow.com/questions/553303/generate-a-random-date-between-two-other-dates + """ + + delta = end - start + int_delta = (delta.days * 24 * 60 * 60) + delta.seconds + random_second = randrange(int_delta) + return start + timedelta(seconds=random_second) + + + +if __name__ == "__main__": + write_schema_examples(out_dir=EXAMPLE_DIR, generate_random=True) From dba9e26c014dc4233b17d811f41e4107bdb996dc Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Thu, 6 Oct 2022 22:50:42 -0700 Subject: [PATCH 05/21] Initial example data. 1. Add example data (imperfect for now) generated by script `create_example` in data/example/data 2. Update validate-data GH action to refer to correct repos. --- .github/workflows/validate-data.yml | 8 +- README.md | 25 ++++++ data/example/README.md | 16 ++++ data/example/data/datapackage.json | 79 +++++++++++++++++++ data/example/data/devices.csv | 6 ++ data/example/data/fare_transactions.csv | 6 ++ data/example/data/operators.csv | 6 ++ data/example/data/passenger_events.csv | 6 ++ data/example/data/station_activities.csv | 6 ++ data/example/data/stop_visits.csv | 6 ++ data/example/data/train_cars.csv | 6 ++ data/example/data/trips_performed.csv | 6 ++ data/example/data/vehicle_locations.csv | 6 ++ data/example/data/vehicle_train_cars.csv | 6 ++ data/example/data/vehicles.csv | 6 ++ .../scripts/create_example.py | 74 +++++++++-------- data/example/scripts/requirements.txt | 2 + 17 files changed, 236 insertions(+), 34 deletions(-) create mode 100644 data/example/README.md create mode 100644 data/example/data/datapackage.json create mode 100644 data/example/data/devices.csv create mode 100644 data/example/data/fare_transactions.csv create mode 100644 data/example/data/operators.csv create mode 100644 data/example/data/passenger_events.csv create mode 100644 data/example/data/station_activities.csv create mode 100644 data/example/data/stop_visits.csv create mode 100644 data/example/data/train_cars.csv create mode 100644 data/example/data/trips_performed.csv create mode 100644 data/example/data/vehicle_locations.csv create mode 100644 data/example/data/vehicle_train_cars.csv create mode 100644 data/example/data/vehicles.csv rename data/{my-agency-name => example}/scripts/create_example.py (58%) create mode 100644 data/example/scripts/requirements.txt diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index dbe7db4a..c2e4c61e 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -3,10 +3,12 @@ name: Validate Example TIDES Data on: push: paths: - - 'data/*/TIDES/*' + - 'data/*/data/*' + - 'spec/*' pull_request: paths: - - 'data/*/TIDES/*' + - 'data/*/data/*' + - 'spec/*' workflow_dispatch: create: @@ -19,4 +21,4 @@ jobs: - name: Validate data uses: frictionlessdata/repository@v2 with: - packages: "data/*/TIDES/*.package.y?ml" + packages: "data/*/data/*.package.json" diff --git a/README.md b/README.md index 767ed742..91f0b5bf 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,31 @@ Example data can be found in the `/data` directory, with one directory for each Example data in the `/TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema contained in the respective repository commit. +## Validating TIDES data + +The easiest way to validate data to the TIDES specifications is to use the frictionless framework, which can be installed from the command line using: + +```sh +pip install frictionless +``` + +### Data Package + +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/example/data/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/example/data/datapackage.json). + +Once this is created, mapping the data files to the schema, simply run: + +```sh +frictionless validate datapackage.json +``` + +### Specific files + + +```sh +frictionless validate datapackage.json +``` + ## Contributing to TIDES Those who want to help with the development of the TIDES specification should review the guidance in the [CONTRIBUTING.md](CONTRIBUTING.md) file. diff --git a/data/example/README.md b/data/example/README.md new file mode 100644 index 00000000..5f9b8278 --- /dev/null +++ b/data/example/README.md @@ -0,0 +1,16 @@ +# Example TIDES Data Package + +## Structure + +The structure of a TIDES Data Package follows the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +- `\data`: formatted TIDES data +- `\scripts`: scripts to create, manipulate, or use TIDES data +- `datapackage.json`: data package metadata + +## Validate Data + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` diff --git a/data/example/data/datapackage.json b/data/example/data/datapackage.json new file mode 100644 index 00000000..e27e9e76 --- /dev/null +++ b/data/example/data/datapackage.json @@ -0,0 +1,79 @@ +{ + "name": "example", + "title" : "Example TIDES Data Package", + "licenses" : [ + { + "name": "Apache-2.0" + } + ], + "profile" : "data-package", + "sources" : [ + { + "title": "Randomly-generated Data from /scripts/create_example.py" + } + ], + "contributors": [ + { + "title": "My Name", + "email": "me@myself.com" + } + ], + "maintainers": [ + { + "title": "Another Name", + "email": "another@myself.com" + } + ], + "resources": [ + { + "name": "devices", + "path": "devices.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json" + }, + { + "name": "fare_transactions", + "path": "fare_transactions.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json" + }, + { + "name": "operators", + "path": "operators.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json" + }, + { + "name": "passenger_events", + "path": "passenger_events.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json" + }, + { + "name": "station_activities", + "path": "station_activities.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json" + }, + { + "name": "stop_visits", + "path": "stop_visits.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json" + }, + { + "name": "train_cars", + "path": "train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json" + }, + { + "name": "trips_performed", + "path": "trips_performed.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json" + }, + { + "name": "vehicle_locations", + "path": "vehicle_locations.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json" + }, + { + "name": "vehicles", + "path": "vehicles.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json" + } + ] +} \ No newline at end of file diff --git a/data/example/data/devices.csv b/data/example/data/devices.csv new file mode 100644 index 00000000..e2aeb2c9 --- /dev/null +++ b/data/example/data/devices.csv @@ -0,0 +1,6 @@ +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location +e,s,c,c,Other,e,x,e +n,v,o,a,AFC,h,o,o +f,q,e,h,Other,d,n,f +h,n,s,h,Other,e,o,w +h,n,h,q,AFC,y,w,m diff --git a/data/example/data/fare_transactions.csv b/data/example/data/fare_transactions.csv new file mode 100644 index 00000000..8e73fedc --- /dev/null +++ b/data/example/data/fare_transactions.csv @@ -0,0 +1,6 @@ +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance +n,2022-01-07,00:50:00Z,44.0,x,Activate,m,33,g,u,r,v,39,Optical scan,k,w,b,False,g,i,91.0 +j,2022-01-18,00:53:00Z,33.0,w,Adjust,a,89,e,y,s,h,14,Cash or coins,g,m,o,False,i,k,39.0 +o,2022-01-03,00:10:00Z,33.0,m,Enter,o,77,c,f,t,h,51,Button pressed by driver or operator to indicate a boarding or alighting passenger.,z,v,e,True,b,b,91.0 +p,2022-02-05,00:02:00Z,10.0,l,Enter,r,2,i,l,c,b,33,Optical scan,p,e,b,False,w,y,60.0 +z,2022-03-31,02:15:00Z,82.0,s,Exit,h,74,x,h,f,r,3,Smart card or ticket,v,f,s,True,i,o,24.0 diff --git a/data/example/data/operators.csv b/data/example/data/operators.csv new file mode 100644 index 00000000..490acd9b --- /dev/null +++ b/data/example/data/operators.csv @@ -0,0 +1,6 @@ +operator_id +r +l +h +m +y diff --git a/data/example/data/passenger_events.csv b/data/example/data/passenger_events.csv new file mode 100644 index 00000000..b8b9cd48 --- /dev/null +++ b/data/example/data/passenger_events.csv @@ -0,0 +1,6 @@ +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id +m,2022-01-29,01:59:00Z,b,18,Individual bike alighted,i,j,t,i +s,2022-02-15,02:51:00Z,k,86,Kneel was engaged,f,u,z,l +m,2022-02-24,01:08:00Z,c,12,Lift was raised,q,z,h,f +i,2022-03-15,00:09:00Z,g,98,Door opened,j,h,r,x +j,2022-02-14,01:49:00Z,n,96,Passenger boarded,i,o,y,m diff --git a/data/example/data/station_activities.csv b/data/example/data/station_activities.csv new file mode 100644 index 00000000..c6c183c5 --- /dev/null +++ b/data/example/data/station_activities.csv @@ -0,0 +1,6 @@ +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits +2022-03-15,d,2022-01-01T02:38:00Z,2022-01-01T00:38:00Z,k,13,22,61,27.0,65.0,67.0,86.0,68.0,69.0,45.0,58.0,35.0,11.0,59.0,19.0,61.0,24.0,91.0,68.0,55,97,48,35 +2022-02-23,k,2022-01-01T02:11:00Z,2022-01-01T00:38:00Z,r,79,63,53,85.0,63.0,6.0,23.0,12.0,11.0,4.0,94.0,14.0,80.0,17.0,95.0,76.0,68.0,81.0,86.0,25,72,1,51 +2022-02-21,s,2022-01-01T00:07:00Z,2022-01-01T01:16:00Z,n,48,78,72,62.0,19.0,43.0,42.0,87.0,69.0,69.0,40.0,78.0,72.0,76.0,69.0,49.0,12.0,67.0,59.0,22,81,9,13 +2022-01-26,o,2022-01-01T02:09:00Z,2022-01-01T00:10:00Z,d,97,96,25,29.0,14.0,41.0,62.0,70.0,68.0,47.0,17.0,23.0,95.0,23.0,31.0,5.0,59.0,5.0,13.0,37,14,79,28 +2022-01-18,v,2022-01-01T02:09:00Z,2022-01-01T00:10:00Z,s,75,15,58,54.0,24.0,40.0,90.0,28.0,91.0,47.0,82.0,65.0,68.0,75.0,21.0,88.0,41.0,38.0,59.0,7,83,4,76 diff --git a/data/example/data/stop_visits.csv b/data/example/data/stop_visits.csv new file mode 100644 index 00000000..126ff653 --- /dev/null +++ b/data/example/data/stop_visits.csv @@ -0,0 +1,6 @@ +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship +2022-01-23,f,39,b,69,v,False,00:41:00Z,01:39:00Z,02:30:00Z,00:17:00Z,74,73,40,17,91,65,02:48:00Z,01:33:00Z,Front door opened and back doors remain closed,92.0,False,92.0,47.0,True,26,90.0,81,3.0,11.0,5.0,16.0,96.0,79.0,53.0,7.0,24.0,6.0,78.0,8.0,33.0,84.0,35.0,87.0,Skipped +2022-01-30,a,57,g,0,w,True,00:44:00Z,02:35:00Z,00:12:00Z,02:05:00Z,81,67,35,90,81,58,00:26:00Z,02:24:00Z,Doors did not open,68.0,True,41.0,76.0,True,97,14.0,46,99.0,4.0,45.0,97.0,88.0,87.0,15.0,57.0,29.0,61.0,87.0,28.0,88.0,51.0,76.0,13.0,Scheduled +2022-03-17,m,75,i,50,n,True,02:45:00Z,02:33:00Z,00:09:00Z,02:35:00Z,61,70,36,10,67,28,01:01:00Z,02:10:00Z,Other configuration,72.0,True,54.0,0.0,False,51,67.0,51,31.0,12.0,28.0,12.0,25.0,66.0,64.0,25.0,64.0,49.0,51.0,80.0,90.0,90.0,84.0,79.0,Canceled +2022-02-10,l,68,x,11,w,True,01:14:00Z,01:26:00Z,02:39:00Z,01:30:00Z,58,47,90,74,10,65,00:22:00Z,01:55:00Z,Back doors opened and front door remained closed,67.0,True,18.0,64.0,True,12,90.0,35,27.0,71.0,45.0,94.0,85.0,40.0,35.0,93.0,23.0,30.0,44.0,15.0,13.0,43.0,24.0,35.0,Canceled +2022-02-22,l,87,x,48,l,False,00:35:00Z,01:38:00Z,02:39:00Z,01:37:00Z,48,94,23,18,48,68,02:26:00Z,02:47:00Z,Other configuration,71.0,True,87.0,61.0,True,79,49.0,63,44.0,49.0,95.0,80.0,59.0,19.0,49.0,74.0,36.0,36.0,69.0,90.0,45.0,74.0,98.0,96.0,Unscheduled diff --git a/data/example/data/train_cars.csv b/data/example/data/train_cars.csv new file mode 100644 index 00000000..275a9b37 --- /dev/null +++ b/data/example/data/train_cars.csv @@ -0,0 +1,6 @@ +train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type +r,k,w,3,47,94,True,7,Trolley +s,d,s,10,16,96,True,1,Train car +e,e,h,74,47,15,False,36,Trolley +o,n,u,30,87,0,True,17,Other +o,u,o,97,47,0,True,47,Train car diff --git a/data/example/data/trips_performed.csv b/data/example/data/trips_performed.csv new file mode 100644 index 00000000..ccff4a36 --- /dev/null +++ b/data/example/data/trips_performed.csv @@ -0,0 +1,6 @@ +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship +2022-03-20,t,q,q,s,Monorail,x,21,c,y,p,n,01:47:00Z,01:47:00Z,01:06:00Z,00:05:00Z,Travel for on-route replacement of breakdown,Duplicated +2022-01-31,x,u,w,p,Subway / Metro,k,11,k,m,q,p,00:49:00Z,02:56:00Z,01:58:00Z,01:21:00Z,Traveling from a service trip,Unscheduled +2022-01-23,r,n,j,i,Funicular,w,60,v,d,d,a,02:13:00Z,02:22:00Z,02:07:00Z,00:59:00Z,In passenger service,Duplicated +2022-03-31,d,m,w,b,Cable tram,v,6,v,p,c,a,02:20:00Z,01:55:00Z,02:37:00Z,00:18:00Z,Travel for on-route replacement of breakdown,Scheduled +2022-01-28,b,i,y,l,Aerial lift,u,77,e,c,c,y,01:56:00Z,01:27:00Z,01:10:00Z,02:55:00Z,Travel for on-route replacement of breakdown,Unscheduled diff --git a/data/example/data/vehicle_locations.csv b/data/example/data/vehicle_locations.csv new file mode 100644 index 00000000..2089b65f --- /dev/null +++ b/data/example/data/vehicle_locations.csv @@ -0,0 +1,6 @@ +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship +k,2022-01-14,00:20:00Z,f,e,w,l,k,In transit to,17.0,120.0,Excellent,243.0,41.0,86,7,18,Other not in service,Skipped +t,2022-01-07,00:58:00Z,n,h,z,g,c,Incoming at,59.0,150.0,Poor,147.0,24.0,58,19,14,In passenger service,Skipped +v,2022-03-05,02:19:00Z,y,e,m,n,q,Stopped at,40.0,-142.0,Good,13.0,46.0,26,36,56,Other not in service,Scheduled +m,2022-02-26,01:06:00Z,j,r,r,x,k,Incoming at,-44.0,163.0,Poor,186.0,45.0,9,5,61,Returning to a garage due to a suspension or breakdown,Skipped +f,2022-02-20,02:45:00Z,v,a,z,f,e,In transit to,47.0,-18.0,Good,337.0,65.0,77,33,14,En-route to service,Unscheduled diff --git a/data/example/data/vehicle_train_cars.csv b/data/example/data/vehicle_train_cars.csv new file mode 100644 index 00000000..bd703099 --- /dev/null +++ b/data/example/data/vehicle_train_cars.csv @@ -0,0 +1,6 @@ +vehicle_id,train_car_id,order,operator_id +e,g,48,h +q,r,35,o +t,l,75,q +j,k,8,h +w,z,5,w diff --git a/data/example/data/vehicles.csv b/data/example/data/vehicles.csv new file mode 100644 index 00000000..81d155dc --- /dev/null +++ b/data/example/data/vehicles.csv @@ -0,0 +1,6 @@ +vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing +y,00:50:00Z,01:59:00Z,u,j,36,13,70,False,42 +s,01:44:00Z,01:21:00Z,f,g,59,94,90,False,9 +k,00:24:00Z,01:39:00Z,m,o,87,0,99,False,84 +b,02:36:00Z,00:51:00Z,i,q,19,86,51,True,53 +x,00:24:00Z,01:02:00Z,l,i,2,12,74,False,1 diff --git a/data/my-agency-name/scripts/create_example.py b/data/example/scripts/create_example.py similarity index 58% rename from data/my-agency-name/scripts/create_example.py rename to data/example/scripts/create_example.py index afab63a9..d654cd4e 100644 --- a/data/my-agency-name/scripts/create_example.py +++ b/data/example/scripts/create_example.py @@ -4,8 +4,9 @@ import json import os import pathlib +import string from random import choice, randrange -from datetime import timedelta +from datetime import timedelta, datetime import numpy as np import pandas as pd @@ -56,11 +57,9 @@ def write_csv_for_schema( """ schema = read_schema(schema_filename) schema_name = pathlib.Path(schema_filename).stem.split(".")[0] - fields_types = { - f["name"]: frictionless_to_pandas_types[f["type"]] for f in schema["fields"] - } - fields_series = {f: pd.Series(dtype=t) for f, t in fields_types.items()} - df = pd.DataFrame(fields_types) + + fields_series = {f['name']: field_to_pd_series(f,example_length) for f in schema["fields"]} + df = pd.DataFrame(fields_series) out_filename = os.path.join(out_dir, schema_name + ".csv") df.to_csv(out_filename, index=False) @@ -68,36 +67,68 @@ def write_csv_for_schema( def field_to_pd_series(field_def: dict, length: int): default_min = 0 default_max = 100 + default_str = list(string.ascii_lowercase) + default_dt_min = 'Jan 1 2022 12:01AM' + default_dt_max = 'Apr 1 2022 3:00AM' + if field_def["type"] in ["number","integer"]: _min = field_def.get("constraints", {}).get("minimum", default_min) - _max = field_def.get("constraints", {}).get("maximum", default_min) + _max = field_def.get("constraints", {}).get("maximum", default_max) return pd.Series( np.random.randint(_min, _max, length), dtype=frictionless_to_pandas_types[field_def["type"]], ) elif field_def["type"] == "float": _min = field_def.get("constraints", {}).get("minimum", default_min) - _max = field_def.get("constraints", {}).get("maximum", default_min) + _max = field_def.get("constraints", {}).get("maximum", default_max) return pd.Series( np.random.rand(_min, _max) * length, dtype=frictionless_to_pandas_types[field_def["type"]], ) elif field_def["type"] == "string": - default_str = "a value" - _enum = field_def.get("constraints", {}).get("enum", [default_str]) + _enum = field_def.get("constraints", {}).get("enum", default_str) return pd.Series( [choice(_enum) for x in range(length)], dtype=frictionless_to_pandas_types[field_def["type"]], ) elif field_def["type"]=="date": - + _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p').date() + _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p').date() + _rand_int = np.random.randint(0, int((_max - _min).days), length).tolist() + return pd.Series( + [_min+timedelta(days=i) for i in _rand_int ], + dtype=frictionless_to_pandas_types[field_def["type"]], + ) elif field_def["type"]=="time": + _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p') + _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p') + _rand_int = np.random.randint(0, int((_max - _min).seconds/60), length).tolist() + _dt_df = pd.Series( + [_min+timedelta(seconds=i*60) for i in _rand_int ], + dtype=frictionless_to_pandas_types[field_def["type"]], + ) + #format as string + return pd.to_datetime(_dt_df).dt.strftime('%H:%M:%SZ') elif field_def["type"]=="datetime": + _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p') + _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p') + _rand_int = np.random.randint(0, int((_max - _min).seconds/60), length).tolist() + _dt_df = pd.Series( + [_min+timedelta(seconds=i*60) for i in _rand_int], + dtype=frictionless_to_pandas_types[field_def["type"]], + ) + #format as ISO-8601 string + return pd.to_datetime(_dt_df).dt.strftime('%Y-%m-%dT%H:%M:%SZ') + elif field_def["type"]=="boolean": + return pd.Series( + [choice([True,False]) for x in range(length)], + dtype=frictionless_to_pandas_types[field_def["type"]], + ) else: - raise ValueError(f'Don't recognize {field_def["type"]') + raise ValueError(f'Dont recognize {field_def["type"]}') def read_schema(schema_file: str) -> dict: @@ -112,24 +143,5 @@ def read_schema(schema_file: str) -> dict: schema = json.load(f) return schema -default_start_datetime = datetime.strptime('1/1/2022 1:30 PM', '%m/%d/%Y %I:%M %p') -default_end_datetime = datetime.strptime('1/9/2022 6:30 PM', '%m/%d/%Y %I:%M %p' - -def random_date(start: datetime.datetime = , end: datetime.datetime): - """ - This function will return a random datetime between two datetime - objects. - - Source: - https://stackoverflow.com/questions/553303/generate-a-random-date-between-two-other-dates - """ - - delta = end - start - int_delta = (delta.days * 24 * 60 * 60) + delta.seconds - random_second = randrange(int_delta) - return start + timedelta(seconds=random_second) - - - if __name__ == "__main__": write_schema_examples(out_dir=EXAMPLE_DIR, generate_random=True) diff --git a/data/example/scripts/requirements.txt b/data/example/scripts/requirements.txt new file mode 100644 index 00000000..2e554ea1 --- /dev/null +++ b/data/example/scripts/requirements.txt @@ -0,0 +1,2 @@ +frictionless +frictionless[pandas] \ No newline at end of file From eaa009ad86124da9256fe1eff2325a0f44ee5607 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 7 Oct 2022 10:42:11 -0700 Subject: [PATCH 06/21] Update Based on Comments - Linted - Auto-generated tabular-data-package datapackage.json from script - Removed random data and the scripts to generate it, kept csv templates - Added readmes and documentation about each example in directory - --- .github/workflows/clean-docs.yml | 2 +- .github/workflows/validate-data.yml | 2 +- README.md | 1 - data/README.md | 24 ++++ data/example/README.md | 16 +-- data/example/data/datapackage.json | 58 ++++---- data/example/data/devices.csv | 5 - data/example/data/fare_transactions.csv | 5 - data/example/data/operators.csv | 5 - data/example/data/passenger_events.csv | 5 - data/example/data/station_activities.csv | 5 - data/example/data/stop_visits.csv | 5 - data/example/data/train_cars.csv | 5 - data/example/data/trips_performed.csv | 5 - data/example/data/vehicle_locations.csv | 5 - data/example/data/vehicle_train_cars.csv | 5 - data/example/data/vehicles.csv | 5 - data/example/scripts/create_example.py | 174 ++++++++++------------- docs/examples.md | 23 +-- main.py | 44 +++++- 20 files changed, 181 insertions(+), 218 deletions(-) create mode 100644 data/README.md diff --git a/.github/workflows/clean-docs.yml b/.github/workflows/clean-docs.yml index 77a99b55..e045b7b5 100644 --- a/.github/workflows/clean-docs.yml +++ b/.github/workflows/clean-docs.yml @@ -22,4 +22,4 @@ jobs: - name: Delete defunct docs versions run: | echo "Deleting ${{ github.event.ref_name }} version from docs" - mike delete --rebase --push ${{ github.event.ref_name }} \ No newline at end of file + mike delete --rebase --push ${{ github.event.ref_name }} diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index c2e4c61e..5d999605 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -1,6 +1,6 @@ name: Validate Example TIDES Data -on: +on: push: paths: - 'data/*/data/*' diff --git a/README.md b/README.md index 91f0b5bf..4f8ae0fe 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ frictionless validate datapackage.json ### Specific files - ```sh frictionless validate datapackage.json ``` diff --git a/data/README.md b/data/README.md new file mode 100644 index 00000000..26cef3a0 --- /dev/null +++ b/data/README.md @@ -0,0 +1,24 @@ +# Data Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +``` +unique-example-name + \data # data to be validated against the TIDES specification + \data\datapackages.json # data package metadata per https://specs.frictionlessdata.io/data-package/ + \raw # data which the agency uses to create TIDES data + \scripts # scripts used to transform raw --> TIDES +``` + +## Data validation + +Data with a valid `datapackage.json` can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Continuous Data Validation + +Example data in the `\data` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch. diff --git a/data/example/README.md b/data/example/README.md index 5f9b8278..0f606853 100644 --- a/data/example/README.md +++ b/data/example/README.md @@ -1,16 +1,16 @@ # Example TIDES Data Package -## Structure +Template directory for example scaffolding and helper scripts. -The structure of a TIDES Data Package follows the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: +## Scripts for Generating Data -- `\data`: formatted TIDES data -- `\scripts`: scripts to create, manipulate, or use TIDES data -- `datapackage.json`: data package metadata +`scripts\create_example.py` has some template code which can help with the following -## Validate Data +- `write_schema_examples()`: will generate blank csvs according to the TIDES schema +- `write_datapackage()` will generate a datapackage.json based on the TIDES schemas and a set of defaults specified in th script. + +To run both (note this replaces the existing files in the directory) ```bash -pip install frictionless -frictionless validate path/to/your/datapackage.json +python data/example/scripts/create_example.py ``` diff --git a/data/example/data/datapackage.json b/data/example/data/datapackage.json index e27e9e76..180c2c5a 100644 --- a/data/example/data/datapackage.json +++ b/data/example/data/datapackage.json @@ -1,15 +1,10 @@ { - "name": "example", - "title" : "Example TIDES Data Package", - "licenses" : [ + "name": "Example TIDES Data Package", + "title": "example", + "profile": "tabular-data-package", + "licenses": [ { "name": "Apache-2.0" - } - ], - "profile" : "data-package", - "sources" : [ - { - "title": "Randomly-generated Data from /scripts/create_example.py" } ], "contributors": [ @@ -30,35 +25,40 @@ "path": "devices.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json" }, + { + "name": "vehicle_locations", + "path": "vehicle_locations.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json" + }, { "name": "fare_transactions", "path": "fare_transactions.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json" }, + { + "name": "train_cars", + "path": "train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json" + }, { "name": "operators", "path": "operators.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json" }, - { - "name": "passenger_events", - "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json" - }, - { - "name": "station_activities", - "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json" - }, { "name": "stop_visits", "path": "stop_visits.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json" }, { - "name": "train_cars", - "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json" + "name": "vehicle_train_cars", + "path": "vehicle_train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json" + }, + { + "name": "vehicles", + "path": "vehicles.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json" }, { "name": "trips_performed", @@ -66,14 +66,14 @@ "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json" }, { - "name": "vehicle_locations", - "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json" + "name": "station_activities", + "path": "station_activities.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json" }, { - "name": "vehicles", - "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json" + "name": "passenger_events", + "path": "passenger_events.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json" } ] -} \ No newline at end of file +} diff --git a/data/example/data/devices.csv b/data/example/data/devices.csv index e2aeb2c9..6540cce5 100644 --- a/data/example/data/devices.csv +++ b/data/example/data/devices.csv @@ -1,6 +1 @@ device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location -e,s,c,c,Other,e,x,e -n,v,o,a,AFC,h,o,o -f,q,e,h,Other,d,n,f -h,n,s,h,Other,e,o,w -h,n,h,q,AFC,y,w,m diff --git a/data/example/data/fare_transactions.csv b/data/example/data/fare_transactions.csv index 8e73fedc..3f65e125 100644 --- a/data/example/data/fare_transactions.csv +++ b/data/example/data/fare_transactions.csv @@ -1,6 +1 @@ transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance -n,2022-01-07,00:50:00Z,44.0,x,Activate,m,33,g,u,r,v,39,Optical scan,k,w,b,False,g,i,91.0 -j,2022-01-18,00:53:00Z,33.0,w,Adjust,a,89,e,y,s,h,14,Cash or coins,g,m,o,False,i,k,39.0 -o,2022-01-03,00:10:00Z,33.0,m,Enter,o,77,c,f,t,h,51,Button pressed by driver or operator to indicate a boarding or alighting passenger.,z,v,e,True,b,b,91.0 -p,2022-02-05,00:02:00Z,10.0,l,Enter,r,2,i,l,c,b,33,Optical scan,p,e,b,False,w,y,60.0 -z,2022-03-31,02:15:00Z,82.0,s,Exit,h,74,x,h,f,r,3,Smart card or ticket,v,f,s,True,i,o,24.0 diff --git a/data/example/data/operators.csv b/data/example/data/operators.csv index 490acd9b..713abf28 100644 --- a/data/example/data/operators.csv +++ b/data/example/data/operators.csv @@ -1,6 +1 @@ operator_id -r -l -h -m -y diff --git a/data/example/data/passenger_events.csv b/data/example/data/passenger_events.csv index b8b9cd48..34932008 100644 --- a/data/example/data/passenger_events.csv +++ b/data/example/data/passenger_events.csv @@ -1,6 +1 @@ passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id -m,2022-01-29,01:59:00Z,b,18,Individual bike alighted,i,j,t,i -s,2022-02-15,02:51:00Z,k,86,Kneel was engaged,f,u,z,l -m,2022-02-24,01:08:00Z,c,12,Lift was raised,q,z,h,f -i,2022-03-15,00:09:00Z,g,98,Door opened,j,h,r,x -j,2022-02-14,01:49:00Z,n,96,Passenger boarded,i,o,y,m diff --git a/data/example/data/station_activities.csv b/data/example/data/station_activities.csv index c6c183c5..cbbc9d94 100644 --- a/data/example/data/station_activities.csv +++ b/data/example/data/station_activities.csv @@ -1,6 +1 @@ date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits -2022-03-15,d,2022-01-01T02:38:00Z,2022-01-01T00:38:00Z,k,13,22,61,27.0,65.0,67.0,86.0,68.0,69.0,45.0,58.0,35.0,11.0,59.0,19.0,61.0,24.0,91.0,68.0,55,97,48,35 -2022-02-23,k,2022-01-01T02:11:00Z,2022-01-01T00:38:00Z,r,79,63,53,85.0,63.0,6.0,23.0,12.0,11.0,4.0,94.0,14.0,80.0,17.0,95.0,76.0,68.0,81.0,86.0,25,72,1,51 -2022-02-21,s,2022-01-01T00:07:00Z,2022-01-01T01:16:00Z,n,48,78,72,62.0,19.0,43.0,42.0,87.0,69.0,69.0,40.0,78.0,72.0,76.0,69.0,49.0,12.0,67.0,59.0,22,81,9,13 -2022-01-26,o,2022-01-01T02:09:00Z,2022-01-01T00:10:00Z,d,97,96,25,29.0,14.0,41.0,62.0,70.0,68.0,47.0,17.0,23.0,95.0,23.0,31.0,5.0,59.0,5.0,13.0,37,14,79,28 -2022-01-18,v,2022-01-01T02:09:00Z,2022-01-01T00:10:00Z,s,75,15,58,54.0,24.0,40.0,90.0,28.0,91.0,47.0,82.0,65.0,68.0,75.0,21.0,88.0,41.0,38.0,59.0,7,83,4,76 diff --git a/data/example/data/stop_visits.csv b/data/example/data/stop_visits.csv index 126ff653..2801b88c 100644 --- a/data/example/data/stop_visits.csv +++ b/data/example/data/stop_visits.csv @@ -1,6 +1 @@ date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship -2022-01-23,f,39,b,69,v,False,00:41:00Z,01:39:00Z,02:30:00Z,00:17:00Z,74,73,40,17,91,65,02:48:00Z,01:33:00Z,Front door opened and back doors remain closed,92.0,False,92.0,47.0,True,26,90.0,81,3.0,11.0,5.0,16.0,96.0,79.0,53.0,7.0,24.0,6.0,78.0,8.0,33.0,84.0,35.0,87.0,Skipped -2022-01-30,a,57,g,0,w,True,00:44:00Z,02:35:00Z,00:12:00Z,02:05:00Z,81,67,35,90,81,58,00:26:00Z,02:24:00Z,Doors did not open,68.0,True,41.0,76.0,True,97,14.0,46,99.0,4.0,45.0,97.0,88.0,87.0,15.0,57.0,29.0,61.0,87.0,28.0,88.0,51.0,76.0,13.0,Scheduled -2022-03-17,m,75,i,50,n,True,02:45:00Z,02:33:00Z,00:09:00Z,02:35:00Z,61,70,36,10,67,28,01:01:00Z,02:10:00Z,Other configuration,72.0,True,54.0,0.0,False,51,67.0,51,31.0,12.0,28.0,12.0,25.0,66.0,64.0,25.0,64.0,49.0,51.0,80.0,90.0,90.0,84.0,79.0,Canceled -2022-02-10,l,68,x,11,w,True,01:14:00Z,01:26:00Z,02:39:00Z,01:30:00Z,58,47,90,74,10,65,00:22:00Z,01:55:00Z,Back doors opened and front door remained closed,67.0,True,18.0,64.0,True,12,90.0,35,27.0,71.0,45.0,94.0,85.0,40.0,35.0,93.0,23.0,30.0,44.0,15.0,13.0,43.0,24.0,35.0,Canceled -2022-02-22,l,87,x,48,l,False,00:35:00Z,01:38:00Z,02:39:00Z,01:37:00Z,48,94,23,18,48,68,02:26:00Z,02:47:00Z,Other configuration,71.0,True,87.0,61.0,True,79,49.0,63,44.0,49.0,95.0,80.0,59.0,19.0,49.0,74.0,36.0,36.0,69.0,90.0,45.0,74.0,98.0,96.0,Unscheduled diff --git a/data/example/data/train_cars.csv b/data/example/data/train_cars.csv index 275a9b37..3b0e604b 100644 --- a/data/example/data/train_cars.csv +++ b/data/example/data/train_cars.csv @@ -1,6 +1 @@ train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type -r,k,w,3,47,94,True,7,Trolley -s,d,s,10,16,96,True,1,Train car -e,e,h,74,47,15,False,36,Trolley -o,n,u,30,87,0,True,17,Other -o,u,o,97,47,0,True,47,Train car diff --git a/data/example/data/trips_performed.csv b/data/example/data/trips_performed.csv index ccff4a36..ba27e909 100644 --- a/data/example/data/trips_performed.csv +++ b/data/example/data/trips_performed.csv @@ -1,6 +1 @@ date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship -2022-03-20,t,q,q,s,Monorail,x,21,c,y,p,n,01:47:00Z,01:47:00Z,01:06:00Z,00:05:00Z,Travel for on-route replacement of breakdown,Duplicated -2022-01-31,x,u,w,p,Subway / Metro,k,11,k,m,q,p,00:49:00Z,02:56:00Z,01:58:00Z,01:21:00Z,Traveling from a service trip,Unscheduled -2022-01-23,r,n,j,i,Funicular,w,60,v,d,d,a,02:13:00Z,02:22:00Z,02:07:00Z,00:59:00Z,In passenger service,Duplicated -2022-03-31,d,m,w,b,Cable tram,v,6,v,p,c,a,02:20:00Z,01:55:00Z,02:37:00Z,00:18:00Z,Travel for on-route replacement of breakdown,Scheduled -2022-01-28,b,i,y,l,Aerial lift,u,77,e,c,c,y,01:56:00Z,01:27:00Z,01:10:00Z,02:55:00Z,Travel for on-route replacement of breakdown,Unscheduled diff --git a/data/example/data/vehicle_locations.csv b/data/example/data/vehicle_locations.csv index 2089b65f..5dff84a7 100644 --- a/data/example/data/vehicle_locations.csv +++ b/data/example/data/vehicle_locations.csv @@ -1,6 +1 @@ location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship -k,2022-01-14,00:20:00Z,f,e,w,l,k,In transit to,17.0,120.0,Excellent,243.0,41.0,86,7,18,Other not in service,Skipped -t,2022-01-07,00:58:00Z,n,h,z,g,c,Incoming at,59.0,150.0,Poor,147.0,24.0,58,19,14,In passenger service,Skipped -v,2022-03-05,02:19:00Z,y,e,m,n,q,Stopped at,40.0,-142.0,Good,13.0,46.0,26,36,56,Other not in service,Scheduled -m,2022-02-26,01:06:00Z,j,r,r,x,k,Incoming at,-44.0,163.0,Poor,186.0,45.0,9,5,61,Returning to a garage due to a suspension or breakdown,Skipped -f,2022-02-20,02:45:00Z,v,a,z,f,e,In transit to,47.0,-18.0,Good,337.0,65.0,77,33,14,En-route to service,Unscheduled diff --git a/data/example/data/vehicle_train_cars.csv b/data/example/data/vehicle_train_cars.csv index bd703099..9044359c 100644 --- a/data/example/data/vehicle_train_cars.csv +++ b/data/example/data/vehicle_train_cars.csv @@ -1,6 +1 @@ vehicle_id,train_car_id,order,operator_id -e,g,48,h -q,r,35,o -t,l,75,q -j,k,8,h -w,z,5,w diff --git a/data/example/data/vehicles.csv b/data/example/data/vehicles.csv index 81d155dc..67ef59e8 100644 --- a/data/example/data/vehicles.csv +++ b/data/example/data/vehicles.csv @@ -1,6 +1 @@ vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing -y,00:50:00Z,01:59:00Z,u,j,36,13,70,False,42 -s,01:44:00Z,01:21:00Z,f,g,59,94,90,False,9 -k,00:24:00Z,01:39:00Z,m,o,87,0,99,False,84 -b,02:36:00Z,00:51:00Z,i,q,19,86,51,True,53 -x,00:24:00Z,01:02:00Z,l,i,2,12,74,False,1 diff --git a/data/example/scripts/create_example.py b/data/example/scripts/create_example.py index d654cd4e..5f9f532a 100644 --- a/data/example/scripts/create_example.py +++ b/data/example/scripts/create_example.py @@ -4,131 +4,103 @@ import json import os import pathlib -import string -from random import choice, randrange -from datetime import timedelta, datetime -import numpy as np -import pandas as pd -EXAMPLE_DIR = os.path.dirname(os.path.abspath(__file__)) -BASE_REPO_DIR = os.path.dirname(os.path.dirname(os.path.dirname(EXAMPLE_DIR))) +EXAMPLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_REPO_DIR = os.path.dirname(os.path.dirname(EXAMPLE_DIR)) TIDES_SPEC = os.path.join(BASE_REPO_DIR, "spec") - SCHEMAS = glob.glob(os.path.join(TIDES_SPEC, "**/*.schema.json"), recursive=True) - -frictionless_to_pandas_types = { - "date": "datetime64[ns]", - "datetime": "datetime64[ns]", - "time": "datetime64[ns]", - "string": "object", - "integer": "int", - "float": "float64", - "number": "float64", - "boolean": "bool", +SCHEMAS_LOC = "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/" + +# DATAPACKAGE.JSON INFORMATION +# Per https://specs.frictionlessdata.io/data-package/ +TITLE = "example" +NAME = "Example TIDES Data Package" +PROFILE = "tabular-data-package" +LICENSES = [{"name": "Apache-2.0"}] +SOURCES = [{"title": "Generated from /scripts/create_example.py"}] +CONTRIBUTORS = [{"title": "My Name", "email": "me@myself.com"}] +MAINTAINERS = [{"title": "Another Name", "email": "another@myself.com"}] +DATAPACKAGE_TEMPLATE = { + "name": NAME, + "title": TITLE, + "profile": PROFILE, + "licenses": LICENSES, + "contributors": CONTRIBUTORS, + "maintainers": MAINTAINERS, + "resources": [], } def write_schema_examples( out_dir: str, schemas: list = SCHEMAS, - generate_random: bool = False, - example_length: int = 5, ) -> None: + """Write blank csvs to out_dir with headings for for each schema in list. + + Args: + out_dir (str): Where blank csvs are written. + schemas (list, optional): List of schemas to generate blank csvs for. Defaults to SCHEMAS. + """ for s in schemas: - write_csv_for_schema( - s, out_dir, generate_random=generate_random, example_length=example_length - ) + write_csv_for_schema(s, out_dir) + + +def write_datapackage( + out_dir: str, + schemas: list = SCHEMAS, + template: dict = DATAPACKAGE_TEMPLATE, +) -> None: + """Write a datapackage.json file in Frictionless data-package format based on list of schemas. + + Args: + out_dir (str): directory where datapackage.json is written. + schemas (list, optional): List of schemas to add to resources list . Defaults to SCHEMAS. + """ + datapackage = template + datapackage["resources"] = [schema_to_resources(s) for s in schemas] + + out_filename = os.path.join(out_dir, "datapackage.json") + with open(out_filename, "w") as outfile: + outfile.write(json.dumps(datapackage, indent=4)) + print(f"Wrote {out_filename}") + json.dumps + + +def schema_to_resources(schema_filename: str) -> dict: + """Transform a schema filename into a frictionless resource for listing in datapackage.json + + Args: + schema_filename (str): Schema file in frictionless format. + + Returns: + dict: object consistent with frictionless data resource specification + """ + schema_filename = pathlib.Path(schema_filename) + name = schema_filename.stem.split(".")[0] + path = name + ".csv" + schema_loc = SCHEMAS_LOC + name + ".schema.json" + + return {"name": name, "path": path, "schema": schema_loc} def write_csv_for_schema( schema_filename: str, out_dir: str, - generate_random: bool = False, - example_length: int = 5, ) -> None: - """Creates blank or randomly filled csvs which comply with a schema. + """Creates blank csvs which comply with a schema. Args: schema_filename (str): Filename with the Frictionless data schema out_dir (str): Where the csv will be written - generate_random (bool): If true, will generate Defaults to False. - example_length (int): If generate_random is True, will generate this number of records. Defaults to 5. """ schema = read_schema(schema_filename) + fields = [s["name"] for s in schema["fields"]] schema_name = pathlib.Path(schema_filename).stem.split(".")[0] - - fields_series = {f['name']: field_to_pd_series(f,example_length) for f in schema["fields"]} - df = pd.DataFrame(fields_series) out_filename = os.path.join(out_dir, schema_name + ".csv") - df.to_csv(out_filename, index=False) - - -def field_to_pd_series(field_def: dict, length: int): - default_min = 0 - default_max = 100 - default_str = list(string.ascii_lowercase) - default_dt_min = 'Jan 1 2022 12:01AM' - default_dt_max = 'Apr 1 2022 3:00AM' - - if field_def["type"] in ["number","integer"]: - _min = field_def.get("constraints", {}).get("minimum", default_min) - _max = field_def.get("constraints", {}).get("maximum", default_max) - return pd.Series( - np.random.randint(_min, _max, length), - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - elif field_def["type"] == "float": - _min = field_def.get("constraints", {}).get("minimum", default_min) - _max = field_def.get("constraints", {}).get("maximum", default_max) - return pd.Series( - np.random.rand(_min, _max) * length, - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - elif field_def["type"] == "string": - _enum = field_def.get("constraints", {}).get("enum", default_str) - return pd.Series( - [choice(_enum) for x in range(length)], - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - elif field_def["type"]=="date": - _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p').date() - _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p').date() - _rand_int = np.random.randint(0, int((_max - _min).days), length).tolist() - return pd.Series( - [_min+timedelta(days=i) for i in _rand_int ], - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - - elif field_def["type"]=="time": - _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p') - _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p') - _rand_int = np.random.randint(0, int((_max - _min).seconds/60), length).tolist() - _dt_df = pd.Series( - [_min+timedelta(seconds=i*60) for i in _rand_int ], - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - #format as string - return pd.to_datetime(_dt_df).dt.strftime('%H:%M:%SZ') - - elif field_def["type"]=="datetime": - _min = datetime.strptime(field_def.get("constraints", {}).get("minimum", default_dt_min), '%b %d %Y %I:%M%p') - _max = datetime.strptime(field_def.get("constraints", {}).get("maximum", default_dt_max), '%b %d %Y %I:%M%p') - _rand_int = np.random.randint(0, int((_max - _min).seconds/60), length).tolist() - _dt_df = pd.Series( - [_min+timedelta(seconds=i*60) for i in _rand_int], - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - #format as ISO-8601 string - return pd.to_datetime(_dt_df).dt.strftime('%Y-%m-%dT%H:%M:%SZ') - elif field_def["type"]=="boolean": - - return pd.Series( - [choice([True,False]) for x in range(length)], - dtype=frictionless_to_pandas_types[field_def["type"]], - ) - else: - raise ValueError(f'Dont recognize {field_def["type"]}') + with open(out_filename, "w") as outfile: + outfile.write(",".join(fields)) + print(f"Wrote {out_filename}") def read_schema(schema_file: str) -> dict: @@ -143,5 +115,7 @@ def read_schema(schema_file: str) -> dict: schema = json.load(f) return schema + if __name__ == "__main__": - write_schema_examples(out_dir=EXAMPLE_DIR, generate_random=True) + write_schema_examples(out_dir=os.path.join(EXAMPLE_DIR, "data")) + write_datapackage(out_dir=os.path.join(EXAMPLE_DIR, "data")) diff --git a/docs/examples.md b/docs/examples.md index 66af3883..c087ca06 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,24 +1,9 @@ -# Examples - -[![Example Data](https://github.com/TIDES-transit/TIDES/actions/workflows/validate-data.yaml/badge.svg)](https://repository.frictionlessdata.io/pages/dashboard.html?user=TIDES-transit&repo=TIDES&flow=validate-data) +# Example Data Example data can be found in the `/data` directory, with one directory for each example. -## Example validation - -Example data in the `\TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema contained in the respective repository commit. - -## Example Directory Organization - -Each example should follow the following directory structure: - -``` -unique-example-name - \TIDES # data to be validated against the TIDES specification - \raw # data which the agency uses to create TIDES data - \scripts # scripts used to transform raw --> TIDES -``` +{{ include_file('data/README.md')}} -## Example List +## Data List -{{ testingme }} +{{ list_examples('data') }} diff --git a/main.py b/main.py index 3799597d..acb0a659 100644 --- a/main.py +++ b/main.py @@ -52,14 +52,16 @@ def include_file(filename: str, start_line: int = 0, end_line: int = None): return content @env.macro - def frictionless_spec(spec_filename:str= None) -> str: + def frictionless_spec(spec_filename: str = None) -> str: if not base_path: base_path = os.path.dirname(os.path.realpath(__file__)) if not docs_path: docs_path = os.path.join(base_path, "docs") if spec_filename is None: - spec_filename = glob.glob(os.path.join(base_path, "**/*.spec.json"), recursive=True)[0] + spec_filename = glob.glob( + os.path.join(base_path, "**/*.spec.json"), recursive=True + )[0] print("Documenting spec_file: ", spec_filename) @@ -77,6 +79,42 @@ def frictionless_spec(spec_filename:str= None) -> str: content_dict={"SPEC": spec_df.to_markdown(index=False)}, ) + @env.macro + def list_examples(data_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + + Args: + data_dir (str):directory to search in + + Returns: + str: markdown-formatted list + """ + data_dir = os.path.join(env.project_dir, data_dir) + examples = [ + d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) + ] + sep = "\n- " + bullet_list = sep + sep.join([_to_md_link(e) for e in examples]) + + example_docs = bullet_list + + for e in examples: + example_title = f"\n## {e.capitalize()}\n\n" + example_docs += example_title + _readme_filename = os.path.join(data_dir, e, "README.md") + if os.path.exists(_readme_filename): + example_docs += include_file(_readme_filename, start_line=2) + else: + example_docs += ( + f"No README.md in example folder {os.path.join(data_dir,e)}.\n" + ) + + return example_docs + + +def _to_md_link(name): + return f"[{name.capitalize()}](#{name})" + def _format_cell_by_type(x, header): if "enum" in header and len(x) > 1: @@ -364,5 +402,3 @@ def repo_to_docs( outline = line # add any fixes that need to happen here. fout.write(outline) - - From af15c9a04ea3b0dfe9c7fe42e230e199dc4fb33e Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 7 Oct 2022 11:00:49 -0700 Subject: [PATCH 07/21] update validate-data workflow to match datapackage.json --- .github/workflows/validate-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index 5d999605..4f3fbb39 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -21,4 +21,4 @@ jobs: - name: Validate data uses: frictionlessdata/repository@v2 with: - packages: "data/*/data/*.package.json" + packages: "data/*/data/datapackage.json" From 3532b15a30201216c84c182961c3377f47bc0aa0 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 7 Oct 2022 13:56:35 -0700 Subject: [PATCH 08/21] Clean up merge fails --- .markdownlint.yaml | 6 + README.md | 13 +- data/README.md | 18 +++ data/example/scripts/requirements.txt | 2 +- docs/index.md | 4 +- docs/requirements.txt | 2 +- main.py | 190 ++++++-------------------- mkdocs.yml | 1 - requirements.txt | 2 +- 9 files changed, 80 insertions(+), 158 deletions(-) diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 55f1ddb8..e150afdc 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -8,5 +8,11 @@ MD007: # Remove line length limit MD013: false +# Remove "fenced code blocks should have language specified" +MD040: false + +# Remove "no inline HTML" +MD033: false + # Remove "no emphasis as heading" MD036: false diff --git a/README.md b/README.md index 564489fe..b0479ea0 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,15 @@ Human-friendlier documentation is auto-generated and available at: ## Example Data -[![Example Data](https://github.com/TIDES-transit/TIDES/actions/workflows/validate-data.yaml/badge.svg)](https://repository.frictionlessdata.io/pages/dashboard.html?user=TIDES-transit&repo=TIDES&flow=validate-data) - Example data can be found in the `/data` directory, with one directory for each example. -Example data in the `/TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema contained in the respective repository commit. - ## Validating TIDES data -The easiest way to validate data to the TIDES specifications is to use the frictionless framework, which can be installed from the command line using: +TIDES data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: -```sh +```bash pip install frictionless +frictionless validate path/to/your/datapackage.json ``` ### Data Package @@ -40,8 +37,10 @@ frictionless validate datapackage.json ### Specific files +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + ```sh -frictionless validate datapackage.json +frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json ``` ## Contributing to TIDES diff --git a/data/README.md b/data/README.md index 26cef3a0..9b9ac51e 100644 --- a/data/README.md +++ b/data/README.md @@ -19,6 +19,24 @@ pip install frictionless frictionless validate path/to/your/datapackage.json ``` +### Data Package + +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/example/data/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/example/data/datapackage.json). + +Once this is created, mapping the data files to the schema, simply run: + +```sh +frictionless validate datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +``` + ### Continuous Data Validation Example data in the `\data` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch. diff --git a/data/example/scripts/requirements.txt b/data/example/scripts/requirements.txt index 2e554ea1..f64d26bd 100644 --- a/data/example/scripts/requirements.txt +++ b/data/example/scripts/requirements.txt @@ -1,2 +1,2 @@ frictionless -frictionless[pandas] \ No newline at end of file +frictionless[pandas] diff --git a/docs/index.md b/docs/index.md index e31d914c..2578b68e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1 +1,3 @@ -{{ include_file('README.md', downshift_h1= False) }} +# TIDES Transit Specification Suite + +{{ include_file('README.md', start_line = 2, downshift_h1= False) }} diff --git a/docs/requirements.txt b/docs/requirements.txt index 58186af2..e3c5b666 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,4 +8,4 @@ mkdocs-material mkdocs-mermaid2-plugin mkdocs-redirects pandas -tabulate \ No newline at end of file +tabulate diff --git a/main.py b/main.py index ef8f086f..f56ad9dd 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ "tables.md": "tables", } + def define_env(env): """ This is the hook for defining variables, macros and filters @@ -26,7 +27,36 @@ def define_env(env): """ @env.macro - def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_line: int = None): + def list_examples(data_dir: str) -> str: + """Outputs a simple list of the directories in a folder in markdown. + Args: + data_dir (str):directory to search in + Returns: + str: markdown-formatted list + """ + data_dir = os.path.join(env.project_dir, data_dir) + examples = [ + d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) + ] + sep = "\n- " + bullet_list = sep + sep.join([_to_md_link(e) for e in examples]) + example_docs = bullet_list + for e in examples: + example_title = f"\n## {e.capitalize()}\n\n" + example_docs += example_title + _readme_filename = os.path.join(data_dir, e, "README.md") + if os.path.exists(_readme_filename): + example_docs += include_file(_readme_filename, start_line=2) + else: + example_docs += ( + f"No README.md in example folder {os.path.join(data_dir,e)}.\n" + ) + return example_docs + + @env.macro + def include_file( + filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None + ): """ Include a file, optionally indicating start_line and end_line. @@ -34,7 +64,8 @@ def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_li args: filename: file to include, relative to the top directory of the documentation project. - downshift_h1: If true, will downshift headings by 1 if h1 heading found. Defaults to True. + downshift_h1: If true, will downshift headings by 1 if h1 heading found. + Defaults to True. start_line (Optional): if included, will start including the file from this line (indexed to 0) end_line (Optional): if included, will stop including at this line (indexed to 0) @@ -56,17 +87,17 @@ def include_file(filename: str, downshift_h1 = True, start_line: int = 0, end_li print(f"???before downshifting! {full_filename}") if md_heading_re[1].search(content) and downshift_h1: print("!!!downshifting!") - content = re.sub(md_heading_re[5],r'#\1\2',content) - content = re.sub(md_heading_re[4],r'#\1\2',content) - content = re.sub(md_heading_re[3],r'#\1\2',content) - content = re.sub(md_heading_re[2],r'#\1\2',content) - content = re.sub(md_heading_re[1],r'#\1\2',content) + content = re.sub(md_heading_re[5], r"#\1\2", content) + content = re.sub(md_heading_re[4], r"#\1\2", content) + content = re.sub(md_heading_re[3], r"#\1\2", content) + content = re.sub(md_heading_re[2], r"#\1\2", content) + content = re.sub(md_heading_re[1], r"#\1\2", content) _filenamebase = env.page.file.url for _find, _replace in FIND_REPLACE.items(): if _filenamebase in _replace: _replace = _replace.replace(_filenamebase, "") - + content = content.replace(_find, _replace) return content @@ -95,7 +126,7 @@ def frictionless_spec( columns=["fullpath", "fullpath_schema", "path", "schema", "name"] ).reset_index() spec_df["name"] = spec_df["name"].apply( - lambda x: f"[`{x}`](tables.md#{x})".replace("_","-") + lambda x: f"[`{x}`](tables.md#{x})".replace("_", "-") ) return spec_df.to_markdown(index=False) @@ -133,6 +164,10 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] +def _to_md_link(name): + return f"[{name.capitalize()}](#{name})" + + def _document_frictionless_schema(schema_filename: str) -> dict: """Documents a single frictionless schema. @@ -352,140 +387,3 @@ def read_config( resource_df.set_index("name", drop=False, inplace=True) return resource_df - - -def document_schemas( - base_path: str = "", - docs_path: str = "", - outfile_name="tables.md", -) -> None: - """Document frictionless table schema files as markdown tables. - - Args: - base_path (str, optional): base path of repo. Defaults to two directories up from this file. - docs_path (str, optional): path where documentation is written. Defaults to "docs" within - the base_path. - outfile_name (str,optional): name of the file (and corresponding template) to write to. - Defaults to architecture.md. - out_path (str, optional): _description_. Defaults to ''. - """ - - if not base_path: - base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - if not docs_path: - docs_path = os.path.join(base_path, "docs") - - # Create markdown with a table for each schema file - - schema_files = glob.glob( - os.path.join(base_path, "**/*.schema.json"), recursive=True - ) - print("Documenting schemas from: {}".format(schema_files)) - - file_schema_markdown = [] - for s in schema_files: - print(f"Documenting Schema: {s}") - spec_name = s.split("/")[-1].split(".")[0] - spec_title = " ".join([x.capitalize() for x in spec_name.split("_")]) - schema = read_schema(s) - if "_table_type" in schema: - table_type = schema["_table_type"] - if table_type not in TABLE_TYPES: - raise Exception( - "{} table_type property of {} does not match known table types".format( - table_type, spec_name - ) - ) - else: - raise Exception("_table_type property missing from {}".format(spec_name)) - schema_md = "\n### {}\n".format(spec_title) - schema_md += "\n*{}*\n".format(s.split("/")[-1]) - if "description" in schema: - schema_md += "\n{}\n".format(schema["description"]) - schema_md += "\n" + _format_primary_key(schema) - schema_md += "\n\n{}\n".format(_list_to_md_table(schema["fields"])) - file_schema_markdown.append( - {"table_type": table_type, "spec_title": spec_title, "schema_md": schema_md} - ) - - file_schema_markdown.sort(key=lambda schema: schema["spec_title"]) - md = "\n\n" - for table_type in TABLE_TYPES: - md += "##{} Tables\n".format(table_type) - for table in file_schema_markdown: - if table["table_type"] == table_type: - md += table["schema_md"] - - _fill_template( - outfile_path=os.path.join(docs_path, outfile_name), - content_dict={"TABLES": md}, - ) - - -def document_spec( - base_path: str = "", - docs_path: str = "", - outfile_name="architecture.md", -) -> None: - """Translate the frictionless .spec file to a markdown table. - - Args: - base_path (str, optional): base path of repo. Defaults to two directories up from this file. - docs_path (str, optional): path where documentation is written. Defaults to "docs" within - the base_path. - outfile_name (str,optional): name of the file (and corresponding template) to write to. - Defaults to architecture.md. - """ - if not base_path: - base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - if not docs_path: - docs_path = os.path.join(base_path, "docs") - - spec_file = glob.glob(os.path.join(base_path, "**/*.spec.json"), recursive=True)[0] - - print("Documenting spec_file: ", spec_file) - - # Generate a table for overall file requirements - spec_df = read_config(spec_file) - spec_df = spec_df.drop( - columns=["fullpath", "fullpath_schema", "path", "schema", "name"] - ).reset_index() - spec_df["name"] = spec_df["name"].apply( - lambda x: "[`{}`](tables.md#{})".format(x, x) - ) - - _fill_template( - outfile_path=os.path.join(docs_path, outfile_name), - content_dict={"SPEC": spec_df.to_markdown(index=False)}, - ) - - -def repo_to_docs( - infile_name, outfile_name: str = None, base_path: str = "", docs_path: str = "" -): - """Copy files from repo to docs (i.e. README). - - Args: - infile_name (str): file to transfer from base repo to documentation directory - outfile_name (str): what to call the outfile in docs dir. Defaults to infile_name. - base_path (str, optional): _description_. Defaults to ''. - docs_path (str, optional): _description_. Defaults to ''. - """ - if not outfile_name: - outfile_name = infile_name - if not base_path: - base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - if not docs_path: - docs_path = os.path.join(base_path, "docs") - - with open(os.path.join(base_path, infile_name), "rt") as fin: - with open(os.path.join(docs_path, outfile_name), "wt") as fout: - for line in fin: - outline = line - # add any fixes that need to happen here. - fout.write(outline) - - -if __name__ == "__main__": - document_spec() - document_schemas() diff --git a/mkdocs.yml b/mkdocs.yml index d946563c..76f12cec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,4 +85,3 @@ nav: - Table Schemas: tables.md - Example Data: examples.md - Development: development.md - \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d8dec895..b09f5df1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ +-r docs/requirements.txt pre-commit --r docs/requirements.txt \ No newline at end of file From 53a05b994b6f59e3acb3321905cef648b17f3235 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 7 Oct 2022 16:42:07 -0700 Subject: [PATCH 09/21] Responses to comments - Rendered samples as a table w/out readmes - Renamed data folders to samples and TIDES - Added metadata to datapackage.json about vendors, github handles, NTDID, agency name and exposed in documentation - Updated documentation for consistency - Renamed 'example' to 'template' to be more consistent with what it is --- .github/workflows/validate-data.yml | 6 +- README.md | 4 +- data/README.md | 42 ----- data/example/data/datapackage.json | 79 -------- data/example/data/operators.csv | 1 - data/example/data/vehicle_train_cars.csv | 1 - data/example/scripts/requirements.txt | 2 - docs/examples.md | 9 - docs/samples.md | 9 + main.py | 70 +++++-- mkdocs.yml | 2 +- samples/README.md | 83 +++++++++ {data/example => samples/template}/README.md | 6 +- samples/template/TIDES/datapackage.json | 171 ++++++++++++++++++ .../template/TIDES}/devices.csv | 2 +- .../template/TIDES}/fare_transactions.csv | 2 +- samples/template/TIDES/operators.csv | 1 + .../template/TIDES}/passenger_events.csv | 2 +- .../template/TIDES}/station_activities.csv | 2 +- .../template/TIDES}/stop_visits.csv | 2 +- .../template/TIDES}/train_cars.csv | 2 +- .../template/TIDES}/trips_performed.csv | 2 +- .../template/TIDES}/vehicle_locations.csv | 2 +- samples/template/TIDES/vehicle_train_cars.csv | 1 + .../template/TIDES}/vehicles.csv | 2 +- .../template/scripts/create_template.py | 40 ++-- samples/template/scripts/requirements.txt | 1 + 27 files changed, 362 insertions(+), 184 deletions(-) delete mode 100644 data/README.md delete mode 100644 data/example/data/datapackage.json delete mode 100644 data/example/data/operators.csv delete mode 100644 data/example/data/vehicle_train_cars.csv delete mode 100644 data/example/scripts/requirements.txt delete mode 100644 docs/examples.md create mode 100644 docs/samples.md create mode 100644 samples/README.md rename {data/example => samples/template}/README.md (68%) create mode 100644 samples/template/TIDES/datapackage.json rename {data/example/data => samples/template/TIDES}/devices.csv (71%) rename {data/example/data => samples/template/TIDES}/fare_transactions.csv (76%) create mode 100644 samples/template/TIDES/operators.csv rename {data/example/data => samples/template/TIDES}/passenger_events.csv (53%) rename {data/example/data => samples/template/TIDES}/station_activities.csv (98%) rename {data/example/data => samples/template/TIDES}/stop_visits.csv (92%) rename {data/example/data => samples/template/TIDES}/train_cars.csv (97%) rename {data/example/data => samples/template/TIDES}/trips_performed.csv (75%) rename {data/example/data => samples/template/TIDES}/vehicle_locations.csv (82%) create mode 100644 samples/template/TIDES/vehicle_train_cars.csv rename {data/example/data => samples/template/TIDES}/vehicles.csv (91%) rename data/example/scripts/create_example.py => samples/template/scripts/create_template.py (78%) create mode 100644 samples/template/scripts/requirements.txt diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index 4f3fbb39..5ed5a194 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -3,11 +3,11 @@ name: Validate Example TIDES Data on: push: paths: - - 'data/*/data/*' + - 'data/*/TIDES/*' - 'spec/*' pull_request: paths: - - 'data/*/data/*' + - 'data/*/TIDES/*' - 'spec/*' workflow_dispatch: create: @@ -21,4 +21,4 @@ jobs: - name: Validate data uses: frictionlessdata/repository@v2 with: - packages: "data/*/data/datapackage.json" + packages: "data/*/TIDES/datapackage.json" diff --git a/README.md b/README.md index b0479ea0..c3d37527 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository provides data schemas and tools to support the access, managemen ## TIDES specification -The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema standards. +The TIDES specification is maintained in the `/spec` sub-directory as a series of JSON tables compatible with the [frictionless data](https://specs.frictionlessdata.io/table-schema/) table schema and [data package](https://specs.frictionlessdata.io/data-package) standards. Human-friendlier documentation is auto-generated and available at: - [Architecture](architecture.md) @@ -27,7 +27,7 @@ frictionless validate path/to/your/datapackage.json ### Data Package -To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/example/data/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/example/data/datapackage.json). +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). Once this is created, mapping the data files to the schema, simply run: diff --git a/data/README.md b/data/README.md deleted file mode 100644 index 9b9ac51e..00000000 --- a/data/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Data Directory Organization - -Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: - -``` -unique-example-name - \data # data to be validated against the TIDES specification - \data\datapackages.json # data package metadata per https://specs.frictionlessdata.io/data-package/ - \raw # data which the agency uses to create TIDES data - \scripts # scripts used to transform raw --> TIDES -``` - -## Data validation - -Data with a valid `datapackage.json` can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: - -```bash -pip install frictionless -frictionless validate path/to/your/datapackage.json -``` - -### Data Package - -To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/example/data/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/example/data/datapackage.json). - -Once this is created, mapping the data files to the schema, simply run: - -```sh -frictionless validate datapackage.json -``` - -### Specific files - -Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: - -```sh -frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json -``` - -### Continuous Data Validation - -Example data in the `\data` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch. diff --git a/data/example/data/datapackage.json b/data/example/data/datapackage.json deleted file mode 100644 index 180c2c5a..00000000 --- a/data/example/data/datapackage.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "name": "Example TIDES Data Package", - "title": "example", - "profile": "tabular-data-package", - "licenses": [ - { - "name": "Apache-2.0" - } - ], - "contributors": [ - { - "title": "My Name", - "email": "me@myself.com" - } - ], - "maintainers": [ - { - "title": "Another Name", - "email": "another@myself.com" - } - ], - "resources": [ - { - "name": "devices", - "path": "devices.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json" - }, - { - "name": "vehicle_locations", - "path": "vehicle_locations.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json" - }, - { - "name": "fare_transactions", - "path": "fare_transactions.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json" - }, - { - "name": "train_cars", - "path": "train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json" - }, - { - "name": "operators", - "path": "operators.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json" - }, - { - "name": "stop_visits", - "path": "stop_visits.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json" - }, - { - "name": "vehicle_train_cars", - "path": "vehicle_train_cars.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json" - }, - { - "name": "vehicles", - "path": "vehicles.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json" - }, - { - "name": "trips_performed", - "path": "trips_performed.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json" - }, - { - "name": "station_activities", - "path": "station_activities.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json" - }, - { - "name": "passenger_events", - "path": "passenger_events.csv", - "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json" - } - ] -} diff --git a/data/example/data/operators.csv b/data/example/data/operators.csv deleted file mode 100644 index 713abf28..00000000 --- a/data/example/data/operators.csv +++ /dev/null @@ -1 +0,0 @@ -operator_id diff --git a/data/example/data/vehicle_train_cars.csv b/data/example/data/vehicle_train_cars.csv deleted file mode 100644 index 9044359c..00000000 --- a/data/example/data/vehicle_train_cars.csv +++ /dev/null @@ -1 +0,0 @@ -vehicle_id,train_car_id,order,operator_id diff --git a/data/example/scripts/requirements.txt b/data/example/scripts/requirements.txt deleted file mode 100644 index f64d26bd..00000000 --- a/data/example/scripts/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -frictionless -frictionless[pandas] diff --git a/docs/examples.md b/docs/examples.md deleted file mode 100644 index c087ca06..00000000 --- a/docs/examples.md +++ /dev/null @@ -1,9 +0,0 @@ -# Example Data - -Example data can be found in the `/data` directory, with one directory for each example. - -{{ include_file('data/README.md')}} - -## Data List - -{{ list_examples('data') }} diff --git a/docs/samples.md b/docs/samples.md new file mode 100644 index 00000000..7b4125ca --- /dev/null +++ b/docs/samples.md @@ -0,0 +1,9 @@ +# Sample Data + +Sample data can be found in the `/samples` directory, with one directory for each data sample. + +{{ include_file('samples/README.md')}} + +## Data List + +{{ list_samples('samples') }} diff --git a/main.py b/main.py index f56ad9dd..21a5a4df 100644 --- a/main.py +++ b/main.py @@ -27,31 +27,32 @@ def define_env(env): """ @env.macro - def list_examples(data_dir: str) -> str: + def list_samples(data_dir: str) -> str: """Outputs a simple list of the directories in a folder in markdown. Args: data_dir (str):directory to search in Returns: str: markdown-formatted list """ + + fields = ["Sample", "Agency", "Resources", "Vendors"] + df = pd.DataFrame([], columns=fields) + data_dir = os.path.join(env.project_dir, data_dir) - examples = [ + samples = [ d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) ] - sep = "\n- " - bullet_list = sep + sep.join([_to_md_link(e) for e in examples]) - example_docs = bullet_list - for e in examples: - example_title = f"\n## {e.capitalize()}\n\n" - example_docs += example_title - _readme_filename = os.path.join(data_dir, e, "README.md") - if os.path.exists(_readme_filename): - example_docs += include_file(_readme_filename, start_line=2) - else: - example_docs += ( - f"No README.md in example folder {os.path.join(data_dir,e)}.\n" - ) - return example_docs + + for s in samples: + df = pd.concat( + [ + df, + pd.DataFrame( + sample_metadata_as_dict(os.path.join(data_dir, s)), index=[0] + ), + ] + ) + return df.to_markdown(index=False) @env.macro def include_file( @@ -84,9 +85,8 @@ def include_file( 4: re.compile(r"(#{4}\s)(.*)"), 5: re.compile(r"(#{5}\s)(.*)"), } - print(f"???before downshifting! {full_filename}") + if md_heading_re[1].search(content) and downshift_h1: - print("!!!downshifting!") content = re.sub(md_heading_re[5], r"#\1\2", content) content = re.sub(md_heading_re[4], r"#\1\2", content) content = re.sub(md_heading_re[3], r"#\1\2", content) @@ -164,8 +164,38 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] -def _to_md_link(name): - return f"[{name.capitalize()}](#{name})" +def sample_metadata_as_dict(dir: str) -> dict: + """Reads datapackage and translates key metadata to dictionary. + + Args: + dir (str): fully qualified directory for sample data + + Returns: + dict: dictionary with data pacakge name, agency, resources and vendors. + """ + dp_filename = os.path.join(dir, "TIDES", "datapackage.json") + if not os.path.isfile(dp_filename): + raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + _vendors = [ + s.get("vendor", None) for r in dp["resources"] for s in r.get("sources", []) + ] + _vendors = list(set(_vendors) - set([None])) + + return { + "Sample": _to_sample_readme_link(dp["name"], dir), + "Agency": dp["agency"], + "Resources": "
  • `" + + "`
  • `".join([r["name"] for r in dp["resources"]]) + + "`
", + "Vendors": "
  • `" + "`
  • `".join(_vendors) + "`
", + } + + +def _to_sample_readme_link(sample_name, folder_dir): + return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" def _document_frictionless_schema(schema_filename: str) -> dict: diff --git a/mkdocs.yml b/mkdocs.yml index 76f12cec..7519dc10 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -83,5 +83,5 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md - - Example Data: examples.md + - Sample Data: samples.md - Development: development.md diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..c5d7a7f1 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,83 @@ +# Data Directory Organization + +Each TIDES Data Package example should follow the following directory structure, consistent with the structure of the [Frictionless Data Package specification](https://specs.frictionlessdata.io/data-package/), including: + +``` +unique-example-name + \TIDES # data to be validated against the TIDES specification + \TIDES\datapackages.json # data package metadata per https://specs.frictionlessdata.io/data-package/ + \raw # data which the agency uses to create TIDES data + \scripts # scripts used to transform raw --> TIDES +``` + +## Adding Examples + +We encourage the addition of examples, but please follow the following guidelines: + +1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. All individual files should be well under 50 MB. +2. *Include Metadata* as specified in [`datapackage.json`](#data-package). +3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. + +## Data Package + +TIDES data packages must include a [`datapackage.json`](https://specs.frictionlessdata.io/data-package/). Key information to include in [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) includes: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | A human-readable title. | Required | +| `name` | Identifier string as a URL-friendly slug. | Required | +| `description` | Short description of data package. | Recommended | +| `agency` | Transit agency name. | Recommended | +| `ntd_id` | ID for the National Transit Database. | Recommended | +| `profile` | Should be `tabular-data-package` | Required | +| `licenses` | Should be `[{"name": "Apache-2.0"}]` to be consistent with this repository | Required | +| `contributors` | Array of data contributors `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `maintainers` | Array of data maintainers `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | +| `resources` | Array of data files included in your package, formated as a [`tabular-data-resource`](#data-resource)| Required | + +### Template + +A `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). + +Once `datapackage.json` is created for your data, you can easily conduct [data validation](#data-validation) using a variet of tools. + +### Data Resource + +Key fields for each [`tabular-data-resource`](https://specs.frictionlessdata.io/tabular-data-resource/) are as follows: + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `name` | Short sluggable name used to refer to data in this file. | Required | +| `path` | Path of the data resource file relative to the `datapackage.json` | Required | +| `schema` | Data schema to use to valdiate the data resource to | Required | +| `sources` | Array of data sources formatted as a [`source`](#data-source) | Recommended | + +### Data Source + +| **Field** | **Description** | **Required** | +| --------- | --------------- | ------------ | +| `title` | Description of the data source. | Required | +| `component` | What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc. | Recommended | +| `product` | What product was used to generate this data (directly or indirectly)? | Recommended | +| `vendor` | What company makes this product? | Recommended | + +## Data validation + +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: + +```bash +pip install frictionless +frictionless validate path/to/your/datapackage.json +``` + +### Specific files + +Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: + +```sh +frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +``` + +### Continuous Data Validation + +Example data in the `\TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch. diff --git a/data/example/README.md b/samples/template/README.md similarity index 68% rename from data/example/README.md rename to samples/template/README.md index 0f606853..1cdaf7f0 100644 --- a/data/example/README.md +++ b/samples/template/README.md @@ -2,9 +2,9 @@ Template directory for example scaffolding and helper scripts. -## Scripts for Generating Data +## Scripts for Generating Template Data -`scripts\create_example.py` has some template code which can help with the following +`scripts\create_template.py` has some template code which can help with the following - `write_schema_examples()`: will generate blank csvs according to the TIDES schema - `write_datapackage()` will generate a datapackage.json based on the TIDES schemas and a set of defaults specified in th script. @@ -12,5 +12,5 @@ Template directory for example scaffolding and helper scripts. To run both (note this replaces the existing files in the directory) ```bash -python data/example/scripts/create_example.py +python samples/template/scripts/create_template.py ``` diff --git a/samples/template/TIDES/datapackage.json b/samples/template/TIDES/datapackage.json new file mode 100644 index 00000000..6e31ab57 --- /dev/null +++ b/samples/template/TIDES/datapackage.json @@ -0,0 +1,171 @@ +{ + "name": "template", + "title": "Template TIDES Data Package Example", + "agency": "Transit Agency Name", + "ntd_id": "1234-56", + "profile": "tabular-data-package", + "licenses": [ + { + "name": "Apache-2.0" + } + ], + "contributors": [ + { + "title": "My Name", + "email": "me@myself.com", + "github": "myhandle" + } + ], + "maintainers": [ + { + "title": "Another Name", + "email": "another@myself.com", + "github": "myhandle" + } + ], + "resources": [ + { + "name": "devices", + "path": "devices.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_locations", + "path": "vehicle_locations.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "fare_transactions", + "path": "fare_transactions.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "train_cars", + "path": "train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "operators", + "path": "operators.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "stop_visits", + "path": "stop_visits.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicle_train_cars", + "path": "vehicle_train_cars.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "vehicles", + "path": "vehicles.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "trips_performed", + "path": "trips_performed.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "station_activities", + "path": "station_activities.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + }, + { + "name": "passenger_events", + "path": "passenger_events.csv", + "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", + "sources": [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product." + } + ] + } + ] +} \ No newline at end of file diff --git a/data/example/data/devices.csv b/samples/template/TIDES/devices.csv similarity index 71% rename from data/example/data/devices.csv rename to samples/template/TIDES/devices.csv index 6540cce5..44f8314f 100644 --- a/data/example/data/devices.csv +++ b/samples/template/TIDES/devices.csv @@ -1 +1 @@ -device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location \ No newline at end of file diff --git a/data/example/data/fare_transactions.csv b/samples/template/TIDES/fare_transactions.csv similarity index 76% rename from data/example/data/fare_transactions.csv rename to samples/template/TIDES/fare_transactions.csv index 3f65e125..f0a10033 100644 --- a/data/example/data/fare_transactions.csv +++ b/samples/template/TIDES/fare_transactions.csv @@ -1 +1 @@ -transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance \ No newline at end of file diff --git a/samples/template/TIDES/operators.csv b/samples/template/TIDES/operators.csv new file mode 100644 index 00000000..e7c166a1 --- /dev/null +++ b/samples/template/TIDES/operators.csv @@ -0,0 +1 @@ +operator_id \ No newline at end of file diff --git a/data/example/data/passenger_events.csv b/samples/template/TIDES/passenger_events.csv similarity index 53% rename from data/example/data/passenger_events.csv rename to samples/template/TIDES/passenger_events.csv index 34932008..8648f62a 100644 --- a/data/example/data/passenger_events.csv +++ b/samples/template/TIDES/passenger_events.csv @@ -1 +1 @@ -passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id \ No newline at end of file diff --git a/data/example/data/station_activities.csv b/samples/template/TIDES/station_activities.csv similarity index 98% rename from data/example/data/station_activities.csv rename to samples/template/TIDES/station_activities.csv index cbbc9d94..ef9ed208 100644 --- a/data/example/data/station_activities.csv +++ b/samples/template/TIDES/station_activities.csv @@ -1 +1 @@ -date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits \ No newline at end of file diff --git a/data/example/data/stop_visits.csv b/samples/template/TIDES/stop_visits.csv similarity index 92% rename from data/example/data/stop_visits.csv rename to samples/template/TIDES/stop_visits.csv index 2801b88c..434bb090 100644 --- a/data/example/data/stop_visits.csv +++ b/samples/template/TIDES/stop_visits.csv @@ -1 +1 @@ -date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship \ No newline at end of file diff --git a/data/example/data/train_cars.csv b/samples/template/TIDES/train_cars.csv similarity index 97% rename from data/example/data/train_cars.csv rename to samples/template/TIDES/train_cars.csv index 3b0e604b..7366a89b 100644 --- a/data/example/data/train_cars.csv +++ b/samples/template/TIDES/train_cars.csv @@ -1 +1 @@ -train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type +train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type \ No newline at end of file diff --git a/data/example/data/trips_performed.csv b/samples/template/TIDES/trips_performed.csv similarity index 75% rename from data/example/data/trips_performed.csv rename to samples/template/TIDES/trips_performed.csv index ba27e909..7dc3393a 100644 --- a/data/example/data/trips_performed.csv +++ b/samples/template/TIDES/trips_performed.csv @@ -1 +1 @@ -date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship \ No newline at end of file diff --git a/data/example/data/vehicle_locations.csv b/samples/template/TIDES/vehicle_locations.csv similarity index 82% rename from data/example/data/vehicle_locations.csv rename to samples/template/TIDES/vehicle_locations.csv index 5dff84a7..abf9c358 100644 --- a/data/example/data/vehicle_locations.csv +++ b/samples/template/TIDES/vehicle_locations.csv @@ -1 +1 @@ -location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship \ No newline at end of file diff --git a/samples/template/TIDES/vehicle_train_cars.csv b/samples/template/TIDES/vehicle_train_cars.csv new file mode 100644 index 00000000..6a4d41ef --- /dev/null +++ b/samples/template/TIDES/vehicle_train_cars.csv @@ -0,0 +1 @@ +vehicle_id,train_car_id,order,operator_id \ No newline at end of file diff --git a/data/example/data/vehicles.csv b/samples/template/TIDES/vehicles.csv similarity index 91% rename from data/example/data/vehicles.csv rename to samples/template/TIDES/vehicles.csv index 67ef59e8..ac9093c4 100644 --- a/data/example/data/vehicles.csv +++ b/samples/template/TIDES/vehicles.csv @@ -1 +1 @@ -vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing +vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing \ No newline at end of file diff --git a/data/example/scripts/create_example.py b/samples/template/scripts/create_template.py similarity index 78% rename from data/example/scripts/create_example.py rename to samples/template/scripts/create_template.py index 5f9f532a..4d26f454 100644 --- a/data/example/scripts/create_example.py +++ b/samples/template/scripts/create_template.py @@ -14,16 +14,30 @@ # DATAPACKAGE.JSON INFORMATION # Per https://specs.frictionlessdata.io/data-package/ -TITLE = "example" -NAME = "Example TIDES Data Package" +TITLE = "Template TIDES Data Package Example" +NAME = "template" +AGENCY = "Transit Agency Name" +NTD_ID = "1234-56" PROFILE = "tabular-data-package" LICENSES = [{"name": "Apache-2.0"}] SOURCES = [{"title": "Generated from /scripts/create_example.py"}] -CONTRIBUTORS = [{"title": "My Name", "email": "me@myself.com"}] -MAINTAINERS = [{"title": "Another Name", "email": "another@myself.com"}] +CONTRIBUTORS = [{"title": "My Name", "email": "me@myself.com", "github": "myhandle"}] +MAINTAINERS = [ + {"title": "Another Name", "email": "another@myself.com", "github": "myhandle"} +] +RESOURCE_SOURCES = [ + { + "title": "Where did data come from?", + "component": "Type of technology component, i.e. `AVL`", + "product": "Product used.", + "vendor": "Vendor selling product.", + } +] DATAPACKAGE_TEMPLATE = { "name": NAME, "title": TITLE, + "agency": AGENCY, + "ntd_id": NTD_ID, "profile": PROFILE, "licenses": LICENSES, "contributors": CONTRIBUTORS, @@ -32,7 +46,7 @@ } -def write_schema_examples( +def write_schema_template( out_dir: str, schemas: list = SCHEMAS, ) -> None: @@ -46,7 +60,7 @@ def write_schema_examples( write_csv_for_schema(s, out_dir) -def write_datapackage( +def write_datapackage_template( out_dir: str, schemas: list = SCHEMAS, template: dict = DATAPACKAGE_TEMPLATE, @@ -63,7 +77,6 @@ def write_datapackage( out_filename = os.path.join(out_dir, "datapackage.json") with open(out_filename, "w") as outfile: outfile.write(json.dumps(datapackage, indent=4)) - print(f"Wrote {out_filename}") json.dumps @@ -80,8 +93,12 @@ def schema_to_resources(schema_filename: str) -> dict: name = schema_filename.stem.split(".")[0] path = name + ".csv" schema_loc = SCHEMAS_LOC + name + ".schema.json" - - return {"name": name, "path": path, "schema": schema_loc} + return { + "name": name, + "path": path, + "schema": schema_loc, + "sources": RESOURCE_SOURCES, + } def write_csv_for_schema( @@ -100,7 +117,6 @@ def write_csv_for_schema( out_filename = os.path.join(out_dir, schema_name + ".csv") with open(out_filename, "w") as outfile: outfile.write(",".join(fields)) - print(f"Wrote {out_filename}") def read_schema(schema_file: str) -> dict: @@ -117,5 +133,5 @@ def read_schema(schema_file: str) -> dict: if __name__ == "__main__": - write_schema_examples(out_dir=os.path.join(EXAMPLE_DIR, "data")) - write_datapackage(out_dir=os.path.join(EXAMPLE_DIR, "data")) + write_schema_template(out_dir=os.path.join(EXAMPLE_DIR, "TIDES")) + write_datapackage_template(out_dir=os.path.join(EXAMPLE_DIR, "TIDES")) diff --git a/samples/template/scripts/requirements.txt b/samples/template/scripts/requirements.txt new file mode 100644 index 00000000..140b95f1 --- /dev/null +++ b/samples/template/scripts/requirements.txt @@ -0,0 +1 @@ +frictionless From 3aadfea8055883a4bbb30a8a06cbc675c038d152 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 10 Oct 2022 10:00:11 -0700 Subject: [PATCH 10/21] Fix validate CLI call --- samples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/README.md b/samples/README.md index c5d7a7f1..f34e20af 100644 --- a/samples/README.md +++ b/samples/README.md @@ -75,7 +75,7 @@ frictionless validate path/to/your/datapackage.json Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: ```sh -frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json ``` ### Continuous Data Validation From 9661da059002dda1bf8e00ad4a404b74a51790d3 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 10 Oct 2022 10:16:39 -0700 Subject: [PATCH 11/21] Responses to comments - changed workflow title to fix link with spaces - moved datapackage.json up one directory, updated docs, updated source refs - added missing --schema flag --- .github/workflows/validate-data.yml | 2 +- README.md | 6 ++--- samples/README.md | 8 +++---- samples/template/{TIDES => }/datapackage.json | 22 +++++++++---------- 4 files changed, 19 insertions(+), 19 deletions(-) rename samples/template/{TIDES => }/datapackage.json (91%) diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index 5ed5a194..d7801367 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -1,4 +1,4 @@ -name: Validate Example TIDES Data +name: Validate-Samples on: push: diff --git a/README.md b/README.md index c3d37527..8ba5a364 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Human-friendlier documentation is auto-generated and available at: ## Example Data -Example data can be found in the `/data` directory, with one directory for each example. +Sample data can be found in the `/samples` directory, with one directory for each example. ## Validating TIDES data @@ -27,7 +27,7 @@ frictionless validate path/to/your/datapackage.json ### Data Package -To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). +To validate a package of TIDES data, you must add a frictionless-compliant [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) alongside your data which describes which files should be validated to which schemas. Most of this can be copied from [`/data/template/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/datapackage.json). Once this is created, mapping the data files to the schema, simply run: @@ -40,7 +40,7 @@ frictionless validate datapackage.json Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: ```sh -frictionless validate vehicles.csv https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json ``` ## Contributing to TIDES diff --git a/samples/README.md b/samples/README.md index f34e20af..fe0a4c59 100644 --- a/samples/README.md +++ b/samples/README.md @@ -4,10 +4,10 @@ Each TIDES Data Package example should follow the following directory structure, ``` unique-example-name - \TIDES # data to be validated against the TIDES specification - \TIDES\datapackages.json # data package metadata per https://specs.frictionlessdata.io/data-package/ - \raw # data which the agency uses to create TIDES data - \scripts # scripts used to transform raw --> TIDES + \TIDES # Required. Data to be validated against the TIDES specification + \datapackages.json # Required. Data package metadata per https://specs.frictionlessdata.io/data-package/ + \raw # Optional. Data which the agency uses to create TIDES data + \scripts # Optional. Scripts used to transform raw --> TIDES ``` ## Adding Examples diff --git a/samples/template/TIDES/datapackage.json b/samples/template/datapackage.json similarity index 91% rename from samples/template/TIDES/datapackage.json rename to samples/template/datapackage.json index 6e31ab57..eb458a57 100644 --- a/samples/template/TIDES/datapackage.json +++ b/samples/template/datapackage.json @@ -26,7 +26,7 @@ "resources": [ { "name": "devices", - "path": "devices.csv", + "path": "TIDES/devices.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", "sources": [ { @@ -39,7 +39,7 @@ }, { "name": "vehicle_locations", - "path": "vehicle_locations.csv", + "path": "TIDES/vehicle_locations.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", "sources": [ { @@ -52,7 +52,7 @@ }, { "name": "fare_transactions", - "path": "fare_transactions.csv", + "path": "TIDES/fare_transactions.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ { @@ -65,7 +65,7 @@ }, { "name": "train_cars", - "path": "train_cars.csv", + "path": "TIDES/train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", "sources": [ { @@ -78,7 +78,7 @@ }, { "name": "operators", - "path": "operators.csv", + "path": "TIDES/operators.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", "sources": [ { @@ -91,7 +91,7 @@ }, { "name": "stop_visits", - "path": "stop_visits.csv", + "path": "TIDES/stop_visits.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", "sources": [ { @@ -104,7 +104,7 @@ }, { "name": "vehicle_train_cars", - "path": "vehicle_train_cars.csv", + "path": "TIDES/vehicle_train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", "sources": [ { @@ -117,7 +117,7 @@ }, { "name": "vehicles", - "path": "vehicles.csv", + "path": "TIDES/vehicles.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", "sources": [ { @@ -130,7 +130,7 @@ }, { "name": "trips_performed", - "path": "trips_performed.csv", + "path": "TIDES/trips_performed.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", "sources": [ { @@ -143,7 +143,7 @@ }, { "name": "station_activities", - "path": "station_activities.csv", + "path": "TIDES/station_activities.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", "sources": [ { @@ -156,7 +156,7 @@ }, { "name": "passenger_events", - "path": "passenger_events.csv", + "path": "TIDES/passenger_events.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", "sources": [ { From 533261956c7dc96f683a656ac7948dda7a58d68c Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 10 Oct 2022 10:39:18 -0700 Subject: [PATCH 12/21] Removed pandas from markdown table writing per request --- main.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/main.py b/main.py index 21a5a4df..cf106afb 100644 --- a/main.py +++ b/main.py @@ -36,23 +36,19 @@ def list_samples(data_dir: str) -> str: """ fields = ["Sample", "Agency", "Resources", "Vendors"] - df = pd.DataFrame([], columns=fields) data_dir = os.path.join(env.project_dir, data_dir) samples = [ d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) ] + md_table = ( + "| *" + "** | **".join(fields) + "* |\n| " + " ----- |" * len(fields) + "\n" + ) + for s in samples: - df = pd.concat( - [ - df, - pd.DataFrame( - sample_metadata_as_dict(os.path.join(data_dir, s)), index=[0] - ), - ] - ) - return df.to_markdown(index=False) + md_table += datapackage_to_row(os.path.join(data_dir, s), fields) + return md_table @env.macro def include_file( @@ -164,16 +160,19 @@ def frictionless_schemas( TABLE_TYPES = ["Event", "Summary", "Supporting"] -def sample_metadata_as_dict(dir: str) -> dict: - """Reads datapackage and translates key metadata to dictionary. +def datapackage_to_row(dir: str, fields: list) -> str: + """Reads datapackage and translates key metadata to a row in a markdown table. + + Right now, only works for fields: "Sample", "Agency", "Resources", "Vendors" Args: dir (str): fully qualified directory for sample data + fields (list): list of fields to return Returns: - dict: dictionary with data pacakge name, agency, resources and vendors. + str: row in a markdown table with name, agency, resources, vendors """ - dp_filename = os.path.join(dir, "TIDES", "datapackage.json") + dp_filename = os.path.join(dir, "datapackage.json") if not os.path.isfile(dp_filename): raise ValueError(f"Can't find datapackage file:\n {dp_filename}.") with open(dp_filename, "r") as dp_file: @@ -184,15 +183,21 @@ def sample_metadata_as_dict(dir: str) -> dict: ] _vendors = list(set(_vendors) - set([None])) - return { + _md_cells = { "Sample": _to_sample_readme_link(dp["name"], dir), "Agency": dp["agency"], - "Resources": "
  • `" - + "`
  • `".join([r["name"] for r in dp["resources"]]) - + "`
", + "Resources": ( + "
  • `" + + "`
  • `".join([r["name"] for r in dp["resources"]]) + + "`
" + ), "Vendors": "
  • `" + "`
  • `".join(_vendors) + "`
", } + md_row = "| " + " | ".join([_md_cells[f] for f in fields]) + " |\n" + + return md_row + def _to_sample_readme_link(sample_name, folder_dir): return f"[{sample_name.capitalize()}]({folder_dir}/README.md)" From e05cd3cd9780c36b94d8c3fd8c1067ed3247aac1 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 10 Oct 2022 10:39:25 -0700 Subject: [PATCH 13/21] pre-commit --- samples/template/TIDES/devices.csv | 2 +- samples/template/TIDES/fare_transactions.csv | 2 +- samples/template/TIDES/operators.csv | 2 +- samples/template/TIDES/passenger_events.csv | 2 +- samples/template/TIDES/station_activities.csv | 2 +- samples/template/TIDES/stop_visits.csv | 2 +- samples/template/TIDES/train_cars.csv | 2 +- samples/template/TIDES/trips_performed.csv | 2 +- samples/template/TIDES/vehicle_locations.csv | 2 +- samples/template/TIDES/vehicle_train_cars.csv | 2 +- samples/template/TIDES/vehicles.csv | 2 +- samples/template/datapackage.json | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/template/TIDES/devices.csv b/samples/template/TIDES/devices.csv index 44f8314f..6540cce5 100644 --- a/samples/template/TIDES/devices.csv +++ b/samples/template/TIDES/devices.csv @@ -1 +1 @@ -device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location \ No newline at end of file +device_id,stop_id,vehicle_id,train_car_id,device_type,device_vendor,device_model,location diff --git a/samples/template/TIDES/fare_transactions.csv b/samples/template/TIDES/fare_transactions.csv index f0a10033..3f65e125 100644 --- a/samples/template/TIDES/fare_transactions.csv +++ b/samples/template/TIDES/fare_transactions.csv @@ -1 +1 @@ -transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance \ No newline at end of file +transaction_id,date,timestamp,amount,currency_type,fare_action,trip_id_performed,stop_sequence,vehicle_id,device_id,fare_id,stop_id,group_size,media_type,rider_category,fare_product,fare_period,fare_capped,fare_media_id,fare_media_id_purchased,balance diff --git a/samples/template/TIDES/operators.csv b/samples/template/TIDES/operators.csv index e7c166a1..713abf28 100644 --- a/samples/template/TIDES/operators.csv +++ b/samples/template/TIDES/operators.csv @@ -1 +1 @@ -operator_id \ No newline at end of file +operator_id diff --git a/samples/template/TIDES/passenger_events.csv b/samples/template/TIDES/passenger_events.csv index 8648f62a..34932008 100644 --- a/samples/template/TIDES/passenger_events.csv +++ b/samples/template/TIDES/passenger_events.csv @@ -1 +1 @@ -passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id \ No newline at end of file +passenger_event_id,date,timestamp,trip_id_performed,stop_sequence,event_type,vehicle_id,device_id,train_car_id,stop_id diff --git a/samples/template/TIDES/station_activities.csv b/samples/template/TIDES/station_activities.csv index ef9ed208..cbbc9d94 100644 --- a/samples/template/TIDES/station_activities.csv +++ b/samples/template/TIDES/station_activities.csv @@ -1 +1 @@ -date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits \ No newline at end of file +date,stop_id,time_period_start,time_period_end,time_period_category,total_entries,total_exits,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,bike_entries,bike_exits,ramp_entries,ramp_exits diff --git a/samples/template/TIDES/stop_visits.csv b/samples/template/TIDES/stop_visits.csv index 434bb090..2801b88c 100644 --- a/samples/template/TIDES/stop_visits.csv +++ b/samples/template/TIDES/stop_visits.csv @@ -1 +1 @@ -date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship \ No newline at end of file +date,trip_id_performed,stop_sequence,vehicle_id,dwell,stop_id,checkpoint,schedule_arrival_time,schedule_departure_time,actual_arrival_time,actual_departure_time,distance,boarding_1,alighting_1,boarding_2,alighting_2,load,door_open,door_close,door_status,ramp_deployed_time,ramp_failure,kneel_deployed_time,lift_deployed_time,bike_rack_deployed,bike_load,revenue,number_of_transactions,transaction_revenue_cash,transaction_revenue_smartcard,transaction_revenue_magcard,transaction_revenue_bankcard,transaction_revenue_nfc,transaction_revenue_optical,transaction_revenue_operator,transaction_revenue_other,transaction_count_cash,transaction_count_smartcard,transaction_count_magcard,transaction_count_bankcard,transaction_count_nfc,transaction_count_optical,transaction_count_operator,transaction_count_other,schedule_relationship diff --git a/samples/template/TIDES/train_cars.csv b/samples/template/TIDES/train_cars.csv index 7366a89b..3b0e604b 100644 --- a/samples/template/TIDES/train_cars.csv +++ b/samples/template/TIDES/train_cars.csv @@ -1 +1 @@ -train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type \ No newline at end of file +train_car_id,model_name,facility_name,capacity_seated,wheelchair_capacity,bike_capacity,bike_rack,capacity_standing,train_car_type diff --git a/samples/template/TIDES/trips_performed.csv b/samples/template/TIDES/trips_performed.csv index 7dc3393a..ba27e909 100644 --- a/samples/template/TIDES/trips_performed.csv +++ b/samples/template/TIDES/trips_performed.csv @@ -1 +1 @@ -date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship \ No newline at end of file +date,trip_id_performed,vehicle_id,trip_id_scheduled,route_id,route_type,shape_id,direction_id,operator_id,block_id,trip_start_stop_id,trip_end_stop_id,schedule_trip_start,schedule_trip_end,actual_trip_start,actual_trip_end,in_service,schedule_relationship diff --git a/samples/template/TIDES/vehicle_locations.csv b/samples/template/TIDES/vehicle_locations.csv index abf9c358..5dff84a7 100644 --- a/samples/template/TIDES/vehicle_locations.csv +++ b/samples/template/TIDES/vehicle_locations.csv @@ -1 +1 @@ -location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship \ No newline at end of file +location_ping_id,date,timestamp,trip_id_performed,stop_sequence,vehicle_id,device_id,stop_id,current_status,latitude,longitude,gps_quality,heading,speed,odometer,schedule_deviation,headway_deviation,in_service,schedule_relationship diff --git a/samples/template/TIDES/vehicle_train_cars.csv b/samples/template/TIDES/vehicle_train_cars.csv index 6a4d41ef..9044359c 100644 --- a/samples/template/TIDES/vehicle_train_cars.csv +++ b/samples/template/TIDES/vehicle_train_cars.csv @@ -1 +1 @@ -vehicle_id,train_car_id,order,operator_id \ No newline at end of file +vehicle_id,train_car_id,order,operator_id diff --git a/samples/template/TIDES/vehicles.csv b/samples/template/TIDES/vehicles.csv index ac9093c4..67ef59e8 100644 --- a/samples/template/TIDES/vehicles.csv +++ b/samples/template/TIDES/vehicles.csv @@ -1 +1 @@ -vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing \ No newline at end of file +vehicle_id,vehicle_start,vehicle_end,model_name,facility_name,capacity_seated,wheelchair_capacity,capacity_bike,bike_rack,capacity_standing diff --git a/samples/template/datapackage.json b/samples/template/datapackage.json index eb458a57..287a659e 100644 --- a/samples/template/datapackage.json +++ b/samples/template/datapackage.json @@ -168,4 +168,4 @@ ] } ] -} \ No newline at end of file +} From 06a8294726c6a775456f22af9ae8890657918aea Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 12 Jul 2023 17:04:57 -0700 Subject: [PATCH 14/21] lint --- .github/workflows/clean-docs.yml | 4 ++-- .github/workflows/docs.yml | 4 ++-- CONTRIBUTING.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/clean-docs.yml b/.github/workflows/clean-docs.yml index e975519b..9b1b2c05 100644 --- a/.github/workflows/clean-docs.yml +++ b/.github/workflows/clean-docs.yml @@ -14,9 +14,9 @@ jobs: echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV" - name: If no github.head_ref, use last chunk of github.ref as Slug if: github.head_ref == '' - run: | + run: | IFS="\/"; - read -a strarr <<< "${{ github.ref }}"; + read -a strarr <<< "${{ github.ref }}"; last_ref="${strarr[${#strarr[*]}-1]}"; echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV" - name: Document which reference diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 55ebb430..166775d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,9 +22,9 @@ jobs: echo "Slug=$(echo "${{ github.head_ref }}" | sed "s/\//\-/g")" >> "$GITHUB_ENV" - name: If no github.head_ref, use last chunk of github.ref as Slug if: github.head_ref == '' - run: | + run: | IFS="\/"; - read -a strarr <<< "${{ github.ref }}"; + read -a strarr <<< "${{ github.ref }}"; last_ref="${strarr[${#strarr[*]}-1]}"; echo "Slug=$(echo $last_ref)" >> "$GITHUB_ENV" - name: Document which reference diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fddf83c5..3fd8a1f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ By making any contribution to the projects, contributors self-certify to the [Co if not report['valid']: print(f"!!! Invalid:{p}") pprint(report) - else: + else: print(f"Valid:{p}") ``` From 102e1c3e4fb372ad2376ededf6a1108b67b84113 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 12 Jul 2023 17:06:52 -0700 Subject: [PATCH 15/21] update/simply script for creating template files - remove datapackage templating (for now) - add CLI --- samples/template/scripts/create_template.py | 105 ------------------ .../template/scripts/create_template_files.py | 69 ++++++++++++ 2 files changed, 69 insertions(+), 105 deletions(-) delete mode 100644 samples/template/scripts/create_template.py create mode 100644 samples/template/scripts/create_template_files.py diff --git a/samples/template/scripts/create_template.py b/samples/template/scripts/create_template.py deleted file mode 100644 index 443da80f..00000000 --- a/samples/template/scripts/create_template.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -import glob -import json -import os -import pathlib - -from frictionless import Schema - - -EXAMPLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -BASE_REPO_DIR = os.path.dirname(os.path.dirname(EXAMPLE_DIR)) -TIDES_SPEC = os.path.join(BASE_REPO_DIR, "spec") -SCHEMAS = glob.glob(os.path.join(TIDES_SPEC, "**/*.schema.json"), recursive=True) -SCHEMAS_LOC = "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/" - -# DATAPACKAGE.JSON INFORMATION -# Per https://specs.frictionlessdata.io/data-package/ -TITLE = "Template TIDES Data Package Example" -NAME = "template" -AGENCY = "Transit Agency Name" -NTD_ID = "1234-56" -PROFILE = "tabular-data-package" -LICENSES = [{"name": "Apache-2.0"}] -SOURCES = [{"title": "Generated from /scripts/create_example.py"}] -CONTRIBUTORS = [{"title": "My Name", "email": "me@myself.com", "github": "myhandle"}] -MAINTAINERS = [ - {"title": "Another Name", "email": "another@myself.com", "github": "myhandle"} -] -RESOURCE_SOURCES = [ - { - "title": "Where did data come from?", - "component": "Type of technology component, i.e. `AVL`", - "product": "Product used.", - "vendor": "Vendor selling product.", - } -] -DATAPACKAGE_TEMPLATE = { - "name": NAME, - "title": TITLE, - "agency": AGENCY, - "ntd_id": NTD_ID, - "profile": PROFILE, - "licenses": LICENSES, - "contributors": CONTRIBUTORS, - "maintainers": MAINTAINERS, - "resources": [], -} - - -def write_schema_template( - out_dir: str, - schemas: list = SCHEMAS, -) -> None: - """Write blank csvs to out_dir with headings for for each schema in list. - - Args: - out_dir (str): Where blank csvs are written. - schemas (list, optional): List of schemas to generate blank csvs for. Defaults to SCHEMAS. - """ - for s in schemas: - write_csv_for_schema(s, out_dir) - - -def write_datapackage_template( - out_dir: str, - schemas: list = SCHEMAS, - template: dict = DATAPACKAGE_TEMPLATE, -) -> None: - """Write a datapackage.json file in Frictionless data-package format based on list of schemas. - - Args: - out_dir (str): directory where datapackage.json is written. - schemas (list, optional): List of schemas to add to resources list . Defaults to SCHEMAS. - """ - datapackage = template - datapackage["resources"] = [schema_to_resources(s) for s in schemas] - - out_filename = os.path.join(out_dir, "datapackage.json") - with open(out_filename, "w") as outfile: - outfile.write(json.dumps(datapackage, indent=4)) - json.dumps - - -def write_csv_for_schema( - schema_filename: str, - out_dir: str, -) -> None: - """Creates blank csvs which comply with a schema. - - Args: - schema_filename (str): Filename with the Frictionless data schema - out_dir (str): Where the csv will be written - """ - _schema = Schema(schema_filename) - fields = [s["name"] for s in schema["fields"]] - _schema_name = pathlib.Path(schema_filename).stem.split(".")[0] - out_filename = os.path.join(out_dir, _schema_name + ".csv") - with open(out_filename, "w") as outfile: - outfile.write(",".join(fields)) - - -if __name__ == "__main__": - write_schema_template(out_dir=os.path.join(EXAMPLE_DIR, "TIDES")) - write_datapackage_template(out_dir=os.path.join(EXAMPLE_DIR, "TIDES")) diff --git a/samples/template/scripts/create_template_files.py b/samples/template/scripts/create_template_files.py new file mode 100644 index 00000000..a9122f36 --- /dev/null +++ b/samples/template/scripts/create_template_files.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +""" +Generates a set of template files using structure and defaults of **schema.json + +--output-path Saves it in the specified output directory (/path/to/output). +If the argument is not provided, it will default to the same directory as the script. + +--schema can specify using a specific schema instead of the default of all +""" + +import argparse +import glob +import logging +import os +import pathlib + +from frictionless import Schema + +USAGE = """ +python samples/template/scripts/create_template_files.py --output-dir /path/to/output +""" + + +SAMPLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_REPO_DIR = os.path.dirname(os.path.dirname(SAMPLE_DIR)) +SCHEMAS = glob.glob( + os.path.join(os.path.join(BASE_REPO_DIR, "spec"), "**/*.schema.json"), + recursive=True, +) +DEFAULT_OUT_PATH = "../TIDES" + + +def write_template_for_schema( + schema_filename: str, + out_dir: str, +) -> None: + """Creates blank csvs which comply with a schema. + + Args: + schema_filename (str): Filename with the Frictionless data schema + out_dir (str): Where the csv will be written + """ + _schema = Schema(schema_filename) + fields = [s.name for s in _schema.fields] + _schema_name = pathlib.Path(schema_filename).stem.split(".")[0] + out_filename = os.path.join(out_dir, _schema_name + ".csv") + with open(out_filename, "w") as outfile: + logging.info(f"Writing: {out_filename}") + outfile.write(",".join(fields)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + parser = argparse.ArgumentParser(description="Generate a template csvs.") + parser.add_argument( + "--schema", + default=SCHEMAS, + help="List of paths to frictionless schemas to generate. Default: all in /spec dir.", + ) + parser.add_argument( + "--output-path", + default=DEFAULT_OUT_PATH, + help="Directory to save the generated files. Default: same directory as script.", + ) + args = parser.parse_args() + + for s in args.schema: + write_template_for_schema(s, args.output_path) From d7a97562487235e58c3c06adc6eea2194ba18761 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Thu, 13 Jul 2023 09:35:31 -0700 Subject: [PATCH 16/21] add recommended fields --- mkdocs.yml | 5 - samples/README.md | 6 +- spec/tides-data-package.json | 1242 ++++++++++++++++++---------------- 3 files changed, 645 insertions(+), 608 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index d5f3eee8..81d9cc98 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,8 +34,6 @@ plugins: theme: | ^(window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? 'dark' : 'light' securityLevel: 'loose' - - mike: - version_selector: true - redirects: redirect_maps: #original relative to /docs : redirect target '../CODE_OF_CONDUCT.md': 'CODE_OF_CONDUCT.md' @@ -43,9 +41,6 @@ plugins: '../contributors.md': 'contributors.md' - search -extra: - version: - provider: mike watch: - CONTRIBUTING.md diff --git a/samples/README.md b/samples/README.md index fe0a4c59..b5a0418f 100644 --- a/samples/README.md +++ b/samples/README.md @@ -5,7 +5,7 @@ Each TIDES Data Package example should follow the following directory structure, ``` unique-example-name \TIDES # Required. Data to be validated against the TIDES specification - \datapackages.json # Required. Data package metadata per https://specs.frictionlessdata.io/data-package/ + \datapackage.json # Required. [`tides-data-package`](#tides-data-package) metadata \raw # Optional. Data which the agency uses to create TIDES data \scripts # Optional. Scripts used to transform raw --> TIDES ``` @@ -15,12 +15,12 @@ unique-example-name We encourage the addition of examples, but please follow the following guidelines: 1. *No large files* This isn't the place to store your data, rather to document some minimal examples. The recommended size is 100-1000 records per file, more if absolutely required to reproduce an issue with the spec. All individual files should be well under 50 MB. -2. *Include Metadata* as specified in [`datapackage.json`](#data-package). +2. *Include Metadata* as specified in [tides-data-package](#data-package). 3. *Include a README.md* in the base folder of your example with an overview so that it can be included in the documentation. ## Data Package -TIDES data packages must include a [`datapackage.json`](https://specs.frictionlessdata.io/data-package/). Key information to include in [`datapackage.json`](https://specs.frictionlessdata.io/data-package/) includes: +TIDES data packages must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) format (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). Key information to include in `datapackage.json` includes: | **Field** | **Description** | **Required** | | --------- | --------------- | ------------ | diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json index 432ef1f5..7b006dc8 100644 --- a/spec/tides-data-package.json +++ b/spec/tides-data-package.json @@ -1,656 +1,698 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "title": "TIDES Data Package", "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", "type": "object", - "$defs": { - "resource-name": { + "required": [ + "title", + "profile", + "resources" + + ], + "recommended":[ + "name", + "description", + "agency", + "ntd_id", + "licenses", + "contributors", + "maintainers" + ], + "properties": { + "profile": { + "enum": [ + "tides-data-package" + ], "propertyOrder": 10, - "title": "Resource Name", - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "title": "Profile", + "description": "The profile of this descriptor.", + "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", "type": "string", - "pattern": "^([-a-z0-9._/])+$", "examples": [ - "fare_transactions", - "train_cars" + "{\n \"profile\": \"tides-data-package\"\n}\n", + "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" ] }, - "path": { + "title": { "propertyOrder": 20, - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path.", + "title": "Title", + "description": "A human-readable title.", "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "/fare_transactions.schema.json", - "https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json" + "{\n \"title\": \"My Package Title\"\n}\n" ] }, - "dialect": { - "propertyOrder": 50, - "title": "CSV Dialect", - "description": "The CSV dialect descriptor.", - "type": [ - "string", - "object" - ], - "required": [ - "delimiter", - "doubleQuote" - ], - "properties": { - "csvddfVersion": { - "title": "CSV Dialect schema version", - "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", - "type": "number", - "default": 1.2, - "examples:": [ - "1.2" - ] - }, - "delimiter": { - "title": "Delimiter", - "description": "A character sequence to use as the field separator.", - "type": "string", - "default": ",", - "examples": [ - ",", - ";" - ] - }, - "doubleQuote": { - "title": "Double Quote", - "description": "Specifies the handling of quotes inside fields.", - "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", - "type": "boolean", - "default": true, - "examples": [ - "true" - ] - }, - "lineTerminator": { - "title": "Line Terminator", - "description": "Specifies the character sequence that must be used to terminate rows.", - "type": "string", - "default": "\r\n", - "examples": [ - "\\r\\n", - "\\n" - ] - }, - "nullSequence": { - "title": "Null Sequence", - "description": "Specifies the null sequence, for example, `\\N`.", - "type": "string", - "examples": [ - "\\N" - ] - }, - "quoteChar": { - "title": "Quote Character", - "description": "Specifies a one-character string to use as the quoting character.", - "type": "string", - "default": "\"", - "examples": [ - "'" - ] - }, - "escapeChar": { - "title": "Escape Character", - "description": "Specifies a one-character string to use as the escape character.", - "type": "string", - "examples": [ - "\\" - ] - }, - "skipInitialSpace": { - "title": "Skip Initial Space", - "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", - "type": "boolean", - "default": false, - "examples": [ - "true" - ] - }, - "header": { - "title": "Header", - "description": "Specifies if the file includes a header row, always as the first row in the file.", - "type": "boolean", - "default": true, - "examples": [ - "true" - ] - }, - "commentChar": { - "title": "Comment Character", - "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", - "type": "string", - "examples": [ - "#" - ] - }, - "caseSensitiveHeader": { - "title": "Case Sensitive Header", - "description": "Specifies if the case of headers is meaningful.", - "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", - "type": "boolean", - "default": false, - "examples": [ - "true" - ] - } - }, - "person": { - "description": "A person related to this project.", - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself.", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "Daniel Tiger" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "daniel@makebelievetrolley.gov" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this contributor.", - "type": "string", - "examples": [ - "Tiger Family" - ] - }, - "role": { - "title": "Role", - "description": "The person's role in the project", - "type": "string", - "default": "Kideriffic contributor" - } - } + "name": { + "propertyOrder": 30, + "title": "Name", + "description": "Short sluggable (https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "{\n \"name\": \"tides-data-package\"\n}\n" + ] }, - "license":{ - "description": "A license describing the legal use of the entity it is associated with.", - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly.", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "LICENSE", - "https://opensource.org/license/apache-2-0" - ] - } - } + "description": { + "propertyOrder": 40, + "format": "textarea", + "title": "Description", + "description": "Short description of TIDES data package.", + "type": "string", + "examples": [ + "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" + ] }, - "resource:tides": { - "required": [ - "path" - ], - "type": "object", - "properties": { - "profile": { - "const": "tabular-data-resource", - "propertyOrder": 10, - "description": "Schema of resource description." - }, - "path": { - "propertyOrder": 30, - "description": "A reference to the data for this resource, as a valid URI string.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "file.csv" - ] - }, - "title": { - "description": "A human-readable title.", - "type": "string", - "examples": [ - "X observations from Y." - ], - "propertyOrder": 50 - }, - "description": { - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "# My Package description\\nAll about my package." - ], - "propertyOrder": 60, - }, - "homepage": { - "propertyOrder": 70, - "description": "The home on the web that is related to this data package.", - "type": "string", - "format": "uri", - "examples": [ - "http://example.com" - ] - }, - + "agency": { + "propertyOrder": 50, + "format": "textarea", + "title": "Agency", + "description": "Transit agency name.", + "type": "string", + "examples": [ + "{\n \"agency\": \"Coos County Area Transit\"\n}\n" + ] }, - "resource:devices":{ - "allOf": [ - { "$ref": "$defs/resource:tides"}, - { - "schema": { - "pattern":"**devices.schema.json" - } - } + "ntd_id": { + "propertyOrder": 60, + "title": "NTD ID", + "description": "ID for the National Transit Database.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "Background on NTD available at https://www.transit.dot.gov/ntd", + "examples": [ + "{\n \"ntd_id\": \"0R02-00307\"\n}\n" ] - } - }, - "allOf": [ - { - "$ref": "https://frictionlessdata.io/schemas/data-package.json" }, - { - "required": [ - "profile", - "resources", - "title" - ], - "properties": { - "profile": { - "const": "tides-data-package", - "propertyOrder": 10, - "description": "The TIDES-specific profile used to validate the TIDES data package. See [Data Package specification](https://specs.frictionlessdata.io/data-package/#profile).", - "type": "string" - }, - "version": { - "pattern": "^([0-9]+).([0-9]+).([0-9]+)$", - "propertyOrder": 11, - "description": "Version (Major, Minor, Patch) identifier", - "example": "0.0.1" - }, - "title": { - "propertyOrder": 20, - "description": "A human-readable title.", - "type": "string", - "examples": [ - "My TIDES Data Package Title" - ] - }, - "name": { - "propertyOrder": 30, - "description": "Short sluggable (https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "2023-XX-tides-data-package" - ] - }, - "description": { - "propertyOrder": 40, - "format": "textarea", - "description": "Short description of TIDES data package.", - "type": "string", - "examples": [ - "My Package description\\nAll about my package." - ] - }, - "agency": { - "propertyOrder": 50, - "description": "Transit agency name.", - "type": "string", - "examples": [ - "Neighborhood of Make Believe Trolley" - ] - }, - "ntd_id": { - "propertyOrder": 60, - "title": "NTD ID", - "description": "ID for the National Transit Database.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "Background on NTD available at https://www.transit.dot.gov/ntd", - "examples": [ - "0R02-00307" - ] + "contributors": { + "propertyOrder": 70, + "title": "Contributors", + "description": "Array of data contributors `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", + "type": "array", + "minItems": 1, + "items": { + "title": "Contributor", + "description": "A contributor to this descriptor.", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "{\n \"title\": \"Frank Julian Sprague\"\n}\n" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "{\n \"email\": \"example@example.com\"\n}\n" + ] + }, + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this contributor.", + "type": "string" + }, + "role": { + "title": "Role", + "description": "The contributor's role in the project", + "type": "string", + "default": "contributor" + } }, - "contributors": { - "propertyOrder": 70, - "description": "Array of data contributors and their role.", - "type": "array", - "minItems": 1, - "items": { - "$ref": "$defs/person" - } + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." + }, + "examples": [ + "{\n \"contributors\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", + "{\n \"contributors\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" + ] + }, + "maintainers": { + "propertyOrder": 80, + "title": "Maintainers", + "description": "Array of data maintainers `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", + "type": "array", + "minItems": 1, + "items": { + "title": "Maintainer", + "description": "A maintainer of this descriptor.", + "properties": { + "name": { + "title": "Name", + "description": "A human-readable name.", + "type": "string", + "examples": [ + "{\n \"name\": \"George Francis Train\"\n}\n" + ] + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "{\n \"email\": \"example@example.com\"\n}\n" + ] + }, + "organization": { + "title": "Organization", + "description": "An organizational affiliation for this maintainer.", + "type": "string" + }, + "role": { + "title": "Role", + "description": "The maintainer's role in the project", + "type": "string", + "default": "maintainer" } }, - "maintainers": { - "propertyOrder": 80, - "title": "Maintainers", - "description": "Array of data maintainers and their role.", - "type": "array", - "minItems": 1, - "items": { - "$ref": "$defs/person" - } + "required": [ + "title" + ], + "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." }, - "licenses": { - "propertyOrder": 90, - "description": "The license(s) under which this package is published.", - "type": "array", - "minItems": 1, - "items": { - "$ref": "$defs/license" + "examples": [ + "{\n \"maintainers\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", + "{\n \"maintainers\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" + ] + }, + "licenses": { + "propertyOrder": 90, + "title": "Licenses", + "description": "The license(s) under which this package is published.", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "A license for this descriptor.", + "type": "object", + "anyOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "path" + ] + } + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$" + }, + "path": { + "title": "Path", + "description": "A fully qualified public URL, or a Unix-style relative file path.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "{\n \"path\": \"/license.md\"\n}\n", + "{\n \"path\": \"https://opensource.org/license/gpl-2-0/\"\n}\n" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "title": { + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "{\n \"title\": \"My Package Title\"\n}\n" + ] + } } }, - "resources": { - "propertyOrder": 100, - "title": "TIDES Data Files", - "description": "Array of data files included in your package, formated as a tabular-data-resource", - "type": "array", - "minItems": 1, - "items": { - "title": "Tabular Data Resource", - "description": "A Tabular Data Resource.", - "type": "object", - "required": [ + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" + ] + }, + "resources": { + "propertyOrder": 100, + "title": "Tabular Data Resources", + "description": "Array of data files included in your package, formated as a tabular-data-resource", + "type": "array", + "minItems": 1, + "items": { + "title": "Tabular Data Resource", + "description": "A Tabular Data Resource.", + "type": "object", + "required": [ + "name", + "path", + "schema" + ], + "properties": { + "profile": { + "enum": [ + "tabular-data-resource" + ], + "propertyOrder": 10, + "title": "Profile", + "description": "Should be `tides-data-package`", + "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", + "type": "string", + "examples": [ + "{\n \"profile\": \"tides-data-package\"\n}\n", + "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" + ] + }, + "name": { + "propertyOrder": 20, + "title": "Name", + "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", + "examples": [ + "{\n \"name\": \"tabular-data-resource\"\n}\n" + ] + }, + "path": { + "propertyOrder": 30, + "title": "Path", + "description": "A reference to the data for this resource, as a valid URI string.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "{\n \"path\": \"file.csv\"\n}\n", + "{\n \"path\": \"http://example.com/file.csv\"\n}\n" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "schema": { + "propertyOrder": 40, + "title": "Table Schema", + "description": "A Table Schema for this resource, compliant with one of the [TIDES](/) table specifications.", + "type": "string", + "required": [ "name", - "path", - "schema" - ], - "properties": { - "profile": { - "enum": [ - "tabular-data-resource" - ], - "propertyOrder": 10, - "title": "Profile", - "description": "Schema of resource description.", - "context": "Every resource descriptor has a profile.", - "type": "string", - "default":"tabular-data-resource" - }, - "name": { - "propertyOrder": 20, - "title": "Name", - "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "tabular-data-resource" - ] + "path" + ], + "properties": { + "name": { + "propertyOrder": 10, + "title": "Name", + "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "type": "string", + "pattern": "^([-a-z0-9._/])+$", + "examples": [ + "{\n \"name\": \"fare_transactions\"\n}\n", + "{\n \"name\": \"train_cars\"\n}\n" + ] + }, + "path": { + "propertyOrder": 20, + "title": "Path", + "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "{\n \"path\": \"/fare_transactions.schema.json\"\n}\n", + "{\n \"path\": \"https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json\"\n}\n" + ] + } }, - "path": { - "propertyOrder": 30, - "title": "Path", - "description": "A reference to the data for this resource, as a valid URI string.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "file.csv" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + "examples": [ + "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" + ] + }, + "title": { + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "{\n \"title\": \"My Package Title\"\n}\n" + ], + "propertyOrder": 50 + }, + "description": { + "title": "Description", + "description": "A text description. Markdown is encouraged.", + "type": "string", + "examples": [ + "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" + ], + "propertyOrder": 60, + "format": "textarea" + }, + "homepage": { + "propertyOrder": 70, + "title": "Home Page", + "description": "The home on the web that is related to this data package.", + "type": "string", + "format": "uri", + "examples": [ + "{\n \"homepage\": \"http://example.com/\"\n}\n" + ] + }, + "sources": { + "propertyOrder": 140, + "options": { + "hidden": true }, - "schema": { - "propertyOrder": 40, - "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with one of the [TIDES](/) table specifications.", - "type": "string", + "title": "Sources", + "description": "The raw sources for this resource.", + "type": "array", + "minItems": 0, + "items": { + "title": "Source", + "description": "A source file.", + "type": "object", "required": [ - "name", - "path" + "title" ], "properties": { - "name": { - "propertyOrder": 10, - "title": "Name", - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", + "title": { + "title": "Title", + "description": "Description of the data source.", "type": "string", - "pattern": "^([-a-z0-9._/])+$", "examples": [ - "fare_transactions", - "train_cars" + "{\n \"title\": \"My Package Title\"\n}\n" + ] + }, + "component": { + "title": "Component", + "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", + "type": "string" + }, + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", + "type": "string" + }, + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" + }, + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" + } + } + } + }, + "licenses": { + "description": "Should be `[{\"name\": \"Apache-2.0\"}]` to be consistent with this repository", + "propertyOrder": 150, + "options": { + "hidden": true + }, + "title": "Licenses", + "type": "array", + "minItems": 1, + "items": { + "title": "License", + "description": "A license for this descriptor.", + "type": "object", + "anyOf": [ + { + "required": [ + "name" ] }, + { + "required": [ + "path" + ] + } + ], + "properties": { + "name": { + "title": "Open Definition license identifier", + "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", + "type": "string", + "pattern": "^([-a-zA-Z0-9._])+$" + }, "path": { - "propertyOrder": 20, "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", + "description": "A fully qualified public URL, or a relative Unix-style file path.", "type": "string", "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "/fare_transactions.schema.json", - "https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json" + "{\n \"path\": \"/license.md\"\n}\n", + "{\n \"path\": \"https://opensource.org/license/apache-2-0/\"\n}\n" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "title": { + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "{\n \"title\": \"My Package Title\"\n}\n" ] } } }, - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "My Package Title" - ], - "propertyOrder": 50 - }, - "description": { - "title": "Description", - "description": "A text description. Markdown is encouraged.", - "type": "string", - "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" - ], - "propertyOrder": 60, - "format": "textarea" - }, - "homepage": { - "propertyOrder": 70, - "title": "Home Page", - "description": "The home on the web that is related to this data package.", - "type": "string", - "format": "uri", - "examples": [ - "http://example.com" - ] - }, - "sources": { - "propertyOrder": 140, - "options": { - "hidden": true + "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", + "examples": [ + "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" + ] + }, + "dialect": { + "propertyOrder": 50, + "title": "CSV Dialect", + "description": "The CSV dialect descriptor.", + "type": [ + "string", + "object" + ], + "required": [ + "delimiter", + "doubleQuote" + ], + "properties": { + "csvddfVersion": { + "title": "CSV Dialect schema version", + "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", + "type": "number", + "default": 1.2, + "examples:": [ + "{\n \"csvddfVersion\": \"1.2\"\n}\n" + ] }, - "title": "Sources", - "description": "The raw sources for this resource.", - "type": "array", - "minItems": 1, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "Description of the data source.", - "type": "string", - "examples": [ - "My Package Title" - ] - }, - "component": { - "title": "Component", - "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", - "type": "string" - }, - "product": { - "title": "Product", - "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string" - }, - "product_version": { - "title": "Product Version", - "description": "Describe the version of the product was used.", - "type": "string" - }, - "vendor": { - "title": "Vendor", - "description": "What company makes this product?", - "type": "string" - } - } - } - }, - "licenses": { - "description": "Should be `[{\"name\": \"Apache-2.0\"}]` to be consistent with this repository", - "propertyOrder": 150, - "options": { - "hidden": true + "delimiter": { + "title": "Delimiter", + "description": "A character sequence to use as the field separator.", + "type": "string", + "default": ",", + "examples": [ + "{\n \"delimiter\": \",\"\n}\n", + "{\n \"delimiter\": \";\"\n}\n" + ] + }, + "doubleQuote": { + "title": "Double Quote", + "description": "Specifies the handling of quotes inside fields.", + "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", + "type": "boolean", + "default": true, + "examples": [ + "{\n \"doubleQuote\": true\n}\n" + ] + }, + "lineTerminator": { + "title": "Line Terminator", + "description": "Specifies the character sequence that must be used to terminate rows.", + "type": "string", + "default": "\r\n", + "examples": [ + "{\n \"lineTerminator\": \"\\r\\n\"\n}\n", + "{\n \"lineTerminator\": \"\\n\"\n}\n" + ] + }, + "nullSequence": { + "title": "Null Sequence", + "description": "Specifies the null sequence, for example, `\\N`.", + "type": "string", + "examples": [ + "{\n \"nullSequence\": \"\\N\"\n}\n" + ] }, - "title": "Licenses", - "type": "array", - "minItems": 1, - "items": { - "$ref": "$defs/license" + "quoteChar": { + "title": "Quote Character", + "description": "Specifies a one-character string to use as the quoting character.", + "type": "string", + "default": "\"", + "examples": [ + "{\n \"quoteChar\": \"'\"\n}\n" + ] + }, + "escapeChar": { + "title": "Escape Character", + "description": "Specifies a one-character string to use as the escape character.", + "type": "string", + "examples": [ + "{\n \"escapeChar\": \"\\\\\"\n}\n" + ] + }, + "skipInitialSpace": { + "title": "Skip Initial Space", + "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", + "type": "boolean", + "default": false, + "examples": [ + "{\n \"skipInitialSpace\": true\n}\n" + ] + }, + "header": { + "title": "Header", + "description": "Specifies if the file includes a header row, always as the first row in the file.", + "type": "boolean", + "default": true, + "examples": [ + "{\n \"header\": true\n}\n" + ] + }, + "commentChar": { + "title": "Comment Character", + "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", + "type": "string", + "examples": [ + "{\n \"commentChar\": \"#\"\n}\n" + ] + }, + "caseSensitiveHeader": { + "title": "Case Sensitive Header", + "description": "Specifies if the case of headers is meaningful.", + "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", + "type": "boolean", + "default": false, + "examples": [ + "{\n \"caseSensitiveHeader\": true\n}\n" + ] } }, - "dialect": {"$ref": "$defs/dialect"}, - "format": { - "propertyOrder": 80, - "description": "The file format of this resource.", - "context": "`csv`, `xls`, `json` are examples of common formats.", - "type": "string", - "default": "csv", - "examples": [ - "xls" - ] - }, - "mediatype": { - "propertyOrder": 90, - "title": "Media Type", - "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", - "type": "string", - "pattern": "^(.+)/(.+)$", - "default": "text/csv", - "examples": [ - "text/csv" - ] + "examples": [ + "{\n \"dialect\": {\n \"delimiter\": \";\"\n }\n}\n", + "{\n \"dialect\": {\n \"delimiter\": \"\\t\",\n \"quoteChar\": \"'\",\n \"commentChar\": \"#\"\n }\n}\n" + ] + }, + "format": { + "propertyOrder": 80, + "title": "Format", + "description": "The file format of this resource.", + "context": "`csv`, `xls`, `json` are examples of common formats.", + "type": "string", + "examples": [ + "{\n \"format\": \"xls\"\n}\n" + ] + }, + "mediatype": { + "propertyOrder": 90, + "title": "Media Type", + "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", + "type": "string", + "pattern": "^(.+)/(.+)$", + "examples": [ + "{\n \"mediatype\": \"text/csv\"\n}\n" + ] + }, + "encoding": { + "propertyOrder": 100, + "title": "Encoding", + "description": "The file encoding of this resource.", + "type": "string", + "default": "utf-8", + "examples": [ + "{\n \"encoding\": \"utf-8\"\n}\n" + ] + }, + "bytes": { + "propertyOrder": 110, + "options": { + "hidden": true }, - "encoding": { - "propertyOrder": 100, - "description": "The file encoding of this resource.", - "type": "string", - "default": "utf-8", - "examples": [ - "utf-8" - ] - }, - "bytes": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Bytes", - "description": "The size of this resource in bytes.", - "type": "integer", - "examples": [ - "2082" - ] + "title": "Bytes", + "description": "The size of this resource in bytes.", + "type": "integer", + "examples": [ + "{\n \"bytes\": 2082\n}\n" + ] + }, + "hash": { + "propertyOrder": 120, + "options": { + "hidden": true }, - "hash": { - "propertyOrder": 120, - "options": { - "hidden": true - }, - "title": "Hash", - "type": "string", - "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", - "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", - "examples": [ - "d25c9c77f588f5dc32059d2da1136c02", - "SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0" - ] - } + "title": "Hash", + "type": "string", + "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", + "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", + "examples": [ + "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", + "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" + ] } } }, - "sources": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "Array of data sources formatted as a source. Recommended to be documented either here at the top-level or for each individual resource", - "type": "array", - "minItems": 0, - "items": { - "title": "Source", - "description": "A source file.", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "My Package Title" - ] - }, - "path": { - "title": "Path", - "description": "A fully qualified URL, or a POSIX file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "file.csv", - "http://example.com/file.csv" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "example@example.com" - ] - } + "examples": [ + "{\n \"resources\": [\n {\n \"name\": \"my-data\",\n \"data\": [\n \"data.csv\"\n ],\n \"schema\": \"tableschema.json\",\n \"mediatype\": \"text/csv\"\n }\n ]\n}\n" + ] + }, + "sources": { + "propertyOrder": 110, + "options": { + "hidden": true + }, + "title": "Sources", + "description": "Array of data sources formatted as a source. Recommended to be documented either here at the top-level or for each individual resource", + "type": "array", + "minItems": 0, + "items": { + "title": "Source", + "description": "A source file.", + "type": "object", + "required": [ + "title" + ], + "properties": { + "title": { + "title": "Title", + "description": "A human-readable title.", + "type": "string", + "examples": [ + "{\n \"title\": \"My Package Title\"\n}\n" + ] + }, + "path": { + "title": "Path", + "description": "A fully qualified URL, or a POSIX file path.", + "type": "string", + "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", + "examples": [ + "{\n \"path\": \"file.csv\"\n}\n", + "{\n \"path\": \"http://example.com/file.csv\"\n}\n" + ], + "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + }, + "email": { + "title": "Email", + "description": "An email address.", + "type": "string", + "format": "email", + "examples": [ + "{\n \"email\": \"example@example.com\"\n}\n" + ] } - }, - "examples": [ - "{\n \"sources\": [\n {\n \"title\": \"World Bank and OECD\",\n \"path\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" - ] - } + } + }, + "examples": [ + "{\n \"sources\": [\n {\n \"title\": \"World Bank and OECD\",\n \"path\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" + ] } } +} From 446ec3441c142ee9e3ae29e9742dfeb1bc088c7a Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Thu, 13 Jul 2023 13:21:17 -0700 Subject: [PATCH 17/21] Add datapackage documentation page Add macro to document datapackage Update datapackage profile to have better examples and fix errata/errors --- docs/datapackage.md | 23 +++ main.py | 108 ++++++++++++- mkdocs.yml | 1 + samples/README.md | 48 +----- spec/tides-data-package.json | 288 ++++++++++++++++++----------------- 5 files changed, 281 insertions(+), 187 deletions(-) create mode 100644 docs/datapackage.md diff --git a/docs/datapackage.md b/docs/datapackage.md new file mode 100644 index 00000000..58a7aa8a --- /dev/null +++ b/docs/datapackage.md @@ -0,0 +1,23 @@ +# Data Package + +TIDES data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json), which is an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/) schema. + +You may create your own `datapackage.json` based on the documentaiton or start with the provided [template](#template), but don't forget to [validate](#validation) it to make sure it is in the correct format! + +## Data Package Format + +{{ frictionless_data_package('spec/tides-data-package.json') }} + +## Tabular Data Resource + +Required and recommended fields for each `tabluar-data-resource` are as follows: + +{{ frictionless_data_package('spec/tides-data-package.json',sub_schema="resources") }} + +## Template + +A `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). + +Once `datapackage.json` is created for your data, you can easily conduct [data validation](#validation) using a variet of tools. + +## Validation diff --git a/main.py b/main.py index cf106afb..8900b862 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,15 @@ import glob import json +import logging import os import pathlib import re -from typing import Union +from typing import Union, Literal import pandas as pd +log = logging.getLogger("mkdocs") + FIND_REPLACE = { # original relative to /docs : redirect target "": "[Contributing Section](development/#CONTRIBUTING)", "(CODE_OF_CONDUCT.md)": "(development/#CODE_OF_CONDUCT)", @@ -97,24 +100,115 @@ def include_file( content = content.replace(_find, _replace) return content + @env.macro + def frictionless_data_package( + data_package_path: str = "**/data-package.json", + sub_schema: str = None, + include: Literal["required", "recommended", "all"] = "recommended", + ) -> str: + """Describes top-level fields of a data package as a markdown table. + + Note: Right now if it finds more than one, it will take the first one. + + Args: + data_package_path (str, optional): location or glob pattern of data package. If glob, + will use the first found match. + sub_schema: if provided, will look for that portion of the schema and use that as + the "top level" + include: specifies the fields to include. Must be one of: + - "required": will only document required fields + - "recommended": will document required and recommended fields + - "all": will document all fields + Defaults to "recommended". + + Returns: a markdown table string + """ + INCLUDE = ["name", "description", "type", "requirement"] + + dp_filename = glob.glob(data_package_path, recursive=True)[0] + + log.info( + f"Documenting spec_file: {dp_filename}.{sub_schema} including {include}" + ) + + with open(dp_filename, "r") as dp_file: + dp = json.loads(dp_file.read()) + + if sub_schema is not None: + if sub_schema in dp["properties"]: + dp = dp["properties"][sub_schema] + elif sub_schema in dp["$defs"]: + dp = dp["$defs"][sub_schema] + else: + return "Cannot find sub-schema {sub_schema} in data package file {dp_filename}" + if dp["type"] == "array": + dp = dp["items"] + log.info(f"dp: \n{dp}") + + _field_names = [] + + if include in ["required", "recommended"]: + _field_names += dp.get("required", []) + if include == "recommended": + _field_names += dp.get("recommended", []) + if include == "all": + _field_names = list(dp["properties"].keys()) + log.info(f"Documenting {len(_field_names)} fields") + if not _field_names: + return "No fields found to document with parameters." + + _fields = [] + for _f in _field_names: + log.info(f"Documenting: {_f}") + _row_entry = {k: v for k, v in dp["properties"][_f].items() if k in INCLUDE} + if _f in dp.get("required", []): + _row_entry["requirement"] = "required" + elif _f in dp.get("recommended", []): + _row_entry["requirement"] = "recommended" + else: + _row_entry["requirement"] = " - " + + _row_entry["name"] = f"`{_f}`" + if dp["properties"][_f].get("enum"): + _row_entry["description"] += "
**Must** be one of:
  • `" + _row_entry["description"] += "`
  • `".join( + dp["properties"][_f]["enum"] + ) + _row_entry["description"] += "`
" + elif dp["properties"][_f].get("const"): + _row_entry[ + "description" + ] += f"
**Must** be: `{dp['properties'][_f]['const']}`" + elif dp["properties"][_f].get("examples"): + _ex = dp["properties"][_f]["examples"][0].replace("\n", "
") + _row_entry["description"] += f"
**Example**:
`{_ex}`" + + _fields.append(_row_entry) + log.info(f"_fields: {_fields}") + + dp_df = pd.DataFrame(_fields, columns=INCLUDE) + dp_md = dp_df.to_markdown(index=False) + + return dp_md + @env.macro def frictionless_spec( spec_path: str = "**/*.spec.json", ) -> str: """Translate the frictionless .spec file to a markdown table. - Note: Right now if it finds more than one, it will take the first one. + Note: Right now if it finds more than one, it will take the first one. If glob, + will use the first found match. Args: - spec_path (str, optional): base path of repo. Defaults to two directories - up from this file. + spec_path (str, optional): location or glob pattern of spec Returns: a markdown table string """ spec_file = glob.glob(spec_path, recursive=True)[0] - print("Documenting spec_file: ", spec_file) + log.info(f"Documenting spec_file: {spec_file}") # Generate a table for overall file requirements spec_df = read_config(spec_file) @@ -141,7 +235,7 @@ def frictionless_schemas( # Create markdown with a table for each schema file schema_files = glob.glob(schema_path, recursive=True) - print("Documenting schemas from: {}".format(schema_files)) + log.info("Documenting schemas from: {schema_files}") file_schema_markdown = [ _document_frictionless_schema(_s) for _s in schema_files @@ -216,7 +310,7 @@ def _document_frictionless_schema(schema_filename: str) -> dict: "schema_md": markdown table documenting fields """ - print(f"Documenting schema from file: {schema_filename}") + log.info(f"Documenting schema from file: {schema_filename}") spec_name = schema_filename.split("/")[-1].split(".")[0] spec_title = " ".join([x.capitalize() for x in spec_name.split("_")]) schema = read_schema(schema_filename) diff --git a/mkdocs.yml b/mkdocs.yml index 81d9cc98..652be641 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -80,5 +80,6 @@ nav: - Home: index.md - Architecture: architecture.md - Table Schemas: tables.md + - Data Package: datapackage.md - Sample Data: samples.md - Development: development.md diff --git a/samples/README.md b/samples/README.md index b5a0418f..3a29a7d7 100644 --- a/samples/README.md +++ b/samples/README.md @@ -20,54 +20,20 @@ We encourage the addition of examples, but please follow the following guideline ## Data Package -TIDES data packages must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) format (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). Key information to include in `datapackage.json` includes: +TIDES sample data must include a `datapackage.json` in the format specified by the [`tides-data-package` json schema](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) (an extension of the [frictionless data package](https://specs.frictionlessdata.io/data-package/)). -| **Field** | **Description** | **Required** | -| --------- | --------------- | ------------ | -| `title` | A human-readable title. | Required | -| `name` | Identifier string as a URL-friendly slug. | Required | -| `description` | Short description of data package. | Recommended | -| `agency` | Transit agency name. | Recommended | -| `ntd_id` | ID for the National Transit Database. | Recommended | -| `profile` | Should be `tabular-data-package` | Required | -| `licenses` | Should be `[{"name": "Apache-2.0"}]` to be consistent with this repository | Required | -| `contributors` | Array of data contributors `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | -| `maintainers` | Array of data maintainers `[{"title": "My Name", "github": "my_handle", "email": "me@myself.com"}]` | Recommended | -| `resources` | Array of data files included in your package, formated as a [`tabular-data-resource`](#data-resource)| Required | +See: -### Template - -A `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). - -Once `datapackage.json` is created for your data, you can easily conduct [data validation](#data-validation) using a variet of tools. - -### Data Resource - -Key fields for each [`tabular-data-resource`](https://specs.frictionlessdata.io/tabular-data-resource/) are as follows: - -| **Field** | **Description** | **Required** | -| --------- | --------------- | ------------ | -| `name` | Short sluggable name used to refer to data in this file. | Required | -| `path` | Path of the data resource file relative to the `datapackage.json` | Required | -| `schema` | Data schema to use to valdiate the data resource to | Required | -| `sources` | Array of data sources formatted as a [`source`](#data-source) | Recommended | - -### Data Source - -| **Field** | **Description** | **Required** | -| --------- | --------------- | ------------ | -| `title` | Description of the data source. | Required | -| `component` | What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc. | Recommended | -| `product` | What product was used to generate this data (directly or indirectly)? | Recommended | -| `vendor` | What company makes this product? | Recommended | +- [Full documentation on the `tides-data-package`]({{ site_url }}/datapackage) +- [Example `tides-data-package` file](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template) ## Data validation -Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoke as follows: +Data with a valid [`datapackage.json`](#data-package) can be easily validated using the [frictionless framework](https://framework.frictionlessdata.io/), which can be installed and invoked as follows: ```bash pip install frictionless -frictionless validate path/to/your/datapackage.json +frictionless validate --schema-sync path/to/your/datapackage.json ``` ### Specific files @@ -80,4 +46,4 @@ frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TI ### Continuous Data Validation -Example data in the `\TIDES` subdirectories is validated upon a push action to the main repository according to the `TIDES` schema posted to the `main` branch. +Sample data in the `\TIDES` subdirectories of each sample is validated upon a push action to the main repository. diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json index 7b006dc8..40078576 100644 --- a/spec/tides-data-package.json +++ b/spec/tides-data-package.json @@ -20,18 +20,12 @@ ], "properties": { "profile": { - "enum": [ - "tides-data-package" - ], + "const": "tides-data-package", "propertyOrder": 10, "title": "Profile", - "description": "The profile of this descriptor.", + "description": "The json-schema profile used to validate this datapackage descriptor.", "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string", - "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" - ] + "type": "string" }, "title": { "propertyOrder": 20, @@ -39,18 +33,18 @@ "description": "A human-readable title.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Make Believe Trolley 1/1/2001-2/1/2001" ] }, "name": { "propertyOrder": 30, "title": "Name", - "description": "Short sluggable (https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", "type": "string", "pattern": "^([-a-z0-9._/])+$", "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", "examples": [ - "{\n \"name\": \"tides-data-package\"\n}\n" + "trolley_2001_1" ] }, "description": { @@ -60,7 +54,7 @@ "description": "Short description of TIDES data package.", "type": "string", "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" + "Raw and processed data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" ] }, "agency": { @@ -70,7 +64,7 @@ "description": "Transit agency name.", "type": "string", "examples": [ - "{\n \"agency\": \"Coos County Area Transit\"\n}\n" + "Make Believe Neighborhood Trolley" ] }, "ntd_id": { @@ -81,25 +75,26 @@ "pattern": "^([-a-z0-9._/])+$", "context": "Background on NTD available at https://www.transit.dot.gov/ntd", "examples": [ - "{\n \"ntd_id\": \"0R02-00307\"\n}\n" + "0R02-00307" ] }, "contributors": { "propertyOrder": 70, "title": "Contributors", - "description": "Array of data contributors `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", + "description": "Array of data contributors.", "type": "array", "minItems": 1, "items": { "title": "Contributor", "description": "A contributor to this descriptor.", + "type": "object", "properties": { "name": { "title": "Name", "description": "A human-readable name.", "type": "string", "examples": [ - "{\n \"title\": \"Frank Julian Sprague\"\n}\n" + "Daniel Tiger" ] }, "email": { @@ -108,13 +103,14 @@ "type": "string", "format": "email", "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" + "daniel@makebelievetrolley.gov" ] }, "organization": { "title": "Organization", "description": "An organizational affiliation for this contributor.", - "type": "string" + "type": "string", + "example": "Make Believe Neighborhood Trolley" }, "role": { "title": "Role", @@ -129,26 +125,26 @@ "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." }, "examples": [ - "{\n \"contributors\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"contributors\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" + "{'name':'Daniel Tiger','email': 'daniel@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'contributor'}" ] }, "maintainers": { "propertyOrder": 80, "title": "Maintainers", - "description": "Array of data maintainers `[{\"title\": \"My Name\", \"github\": \"my_handl\", \"email\": \"me@myself.com\"}]`", + "description": "Array of data maintainers", "type": "array", "minItems": 1, "items": { "title": "Maintainer", "description": "A maintainer of this descriptor.", + "type": "object", "properties": { "name": { "title": "Name", "description": "A human-readable name.", "type": "string", "examples": [ - "{\n \"name\": \"George Francis Train\"\n}\n" + "Henrietta Pussycat" ] }, "email": { @@ -157,7 +153,7 @@ "type": "string", "format": "email", "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" + "henrietta@makebelievetrolley.gov" ] }, "organization": { @@ -178,8 +174,8 @@ "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." }, "examples": [ - "{\n \"maintainers\": [\n {\n \"name\": \"Joe Bloggs\"\n }\n ]\n}\n", - "{\n \"maintainers\": [\n {\n \"name\": \"Miriam Coates\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" + "{'name':'Henrietta Pussycat','email': 'henrietta@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'maintainer'}" + ] }, "licenses": { @@ -190,7 +186,7 @@ "minItems": 1, "items": { "title": "License", - "description": "A license for this descriptor.", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", "type": "object", "anyOf": [ { @@ -209,7 +205,8 @@ "title": "Open Definition license identifier", "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] }, "path": { "title": "Path", @@ -217,30 +214,30 @@ "type": "string", "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/gpl-2-0/\"\n}\n" + "https://opendefinition.org/licenses/odc-by/", + "license.md" ], "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." }, "title": { "title": "Title", - "description": "A human-readable title.", + "description": "A human-readable title of the license.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Creative Commons Attribution 4.0" ] } } }, "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" ] }, "resources": { "propertyOrder": 100, "title": "Tabular Data Resources", - "description": "Array of data files included in your package, formated as a tabular-data-resource", + "description": "Array describing the data files included in your package, formated as a [`tabular-data-resource`](#tabular-data-resource)", "type": "array", "minItems": 1, "items": { @@ -250,32 +247,35 @@ "required": [ "name", "path", - "schema" + "schema", + "profile" ], + "recommended": [ + "title", + "description", + "sources", + "licenses" + ], "properties": { "profile": { - "enum": [ - "tabular-data-resource" - ], + "const": "tabular-data-resource", "propertyOrder": 10, "title": "Profile", - "description": "Should be `tides-data-package`", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", + "description": "Must be `tabular-data-resource`", "type": "string", "examples": [ - "{\n \"profile\": \"tides-data-package\"\n}\n", - "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" + "tabular-data-resource" ] }, "name": { "propertyOrder": 20, "title": "Name", - "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", + "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string for a data table.", "type": "string", "pattern": "^([-a-z0-9._/])+$", "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", "examples": [ - "{\n \"name\": \"tabular-data-resource\"\n}\n" + "vehicle_locations_2001_01" ] }, "path": { @@ -285,16 +285,15 @@ "type": "string", "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" + "event/2001/vehicle_locations_2001_01.csv" ], "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." }, "schema": { "propertyOrder": 40, "title": "Table Schema", - "description": "A Table Schema for this resource, compliant with one of the [TIDES](/) table specifications.", - "type": "string", + "description": "An object describing the TIDES Table Schema for this resource.", + "type": "object", "required": [ "name", "path" @@ -303,12 +302,24 @@ "name": { "propertyOrder": 10, "title": "Name", + "enum": [ + "devices", + "fare_transactions", + "operators", + "station_activities", + "passenger_events", + "stop_visits", + "train_cars", + "trips_performed", + "vehicle_locations", + "vehicle_train_cars", + "vehicles" + ], "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", "type": "string", - "pattern": "^([-a-z0-9._/])+$", "examples": [ - "{\n \"name\": \"fare_transactions\"\n}\n", - "{\n \"name\": \"train_cars\"\n}\n" + "fare_transactions", + "train_cars" ] }, "path": { @@ -318,30 +329,30 @@ "type": "string", "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"path\": \"/fare_transactions.schema.json\"\n}\n", - "{\n \"path\": \"https://github.com/TIDES-transit/TIDES/blob/main/spec/train_cars.schema.json\"\n}\n" + "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", + "fare_transactions.schema.json" ] } }, "examples": [ - "{\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"first_name\",\n \"type\": \"string\"\n \"constraints\": {\n \"required\": true\n }\n },\n {\n \"name\": \"age\",\n \"type\": \"integer\"\n },\n ],\n \"primaryKey\": [\n \"name\"\n ]\n }\n}\n" + "`{'name': 'devices' ,'path': 'https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json'}`" ] }, "title": { "title": "Title", - "description": "A human-readable title.", + "description": "A human-readable title for the data resource.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Neighborhood Trolley Vehicle Locations: 1/1/2001-2/1/2001" ], "propertyOrder": 50 }, "description": { "title": "Description", - "description": "A text description. Markdown is encouraged.", + "description": "D", "type": "string", "examples": [ - "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" + "Vehicle location event data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" ], "propertyOrder": 60, "format": "textarea" @@ -353,7 +364,7 @@ "type": "string", "format": "uri", "examples": [ - "{\n \"homepage\": \"http://example.com/\"\n}\n" + "http://example.com" ] }, "sources": { @@ -362,12 +373,14 @@ "hidden": true }, "title": "Sources", - "description": "The raw sources for this resource.", + "description": "The raw sources for this resource which describe where the data came from.", "type": "array", "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], "items": { "title": "Source", - "description": "A source file.", "type": "object", "required": [ "title" @@ -375,21 +388,29 @@ "properties": { "title": { "title": "Title", - "description": "Description of the data source.", + "description": "Human-readable title of the data source.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Trolley CAD/AVL System" ] }, "component": { "title": "Component", - "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", - "type": "string" + "description": "The technology component used to generate this data (directly or indirectly)", + "type": "string", + "examples": [ + "CAD/AVL", + "AFC", + "APC" + ] }, "product": { "title": "Product", "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string" + "type": "string", + "examples": [ + "TrolleyMaster" + ] }, "product_version": { "title": "Product Version", @@ -405,17 +426,14 @@ } }, "licenses": { - "description": "Should be `[{\"name\": \"Apache-2.0\"}]` to be consistent with this repository", - "propertyOrder": 150, - "options": { - "hidden": true - }, + "propertyOrder": 90, "title": "Licenses", + "description": "The license(s) under which this package is published.", "type": "array", "minItems": 1, "items": { "title": "License", - "description": "A license for this descriptor.", + "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", "type": "object", "anyOf": [ { @@ -434,32 +452,33 @@ "title": "Open Definition license identifier", "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$" + "pattern": "^([-a-zA-Z0-9._])+$", + "examples": ["odc-by","CC-BY-4.0"] }, "path": { "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path.", + "description": "A fully qualified public URL, or a Unix-style relative file path.", "type": "string", "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"path\": \"/license.md\"\n}\n", - "{\n \"path\": \"https://opensource.org/license/apache-2-0/\"\n}\n" + "https://opendefinition.org/licenses/odc-by/", + "license.md" ], "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." }, "title": { "title": "Title", - "description": "A human-readable title.", + "description": "A human-readable title of the license.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Creative Commons Attribution 4.0" ] } } }, "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", "examples": [ - "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ]\n}\n" + "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" ] }, "dialect": { @@ -481,7 +500,7 @@ "type": "number", "default": 1.2, "examples:": [ - "{\n \"csvddfVersion\": \"1.2\"\n}\n" + "1.2" ] }, "delimiter": { @@ -490,8 +509,8 @@ "type": "string", "default": ",", "examples": [ - "{\n \"delimiter\": \",\"\n}\n", - "{\n \"delimiter\": \";\"\n}\n" + "`;`", + "`|`" ] }, "doubleQuote": { @@ -499,10 +518,7 @@ "description": "Specifies the handling of quotes inside fields.", "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", "type": "boolean", - "default": true, - "examples": [ - "{\n \"doubleQuote\": true\n}\n" - ] + "default": true }, "lineTerminator": { "title": "Line Terminator", @@ -510,8 +526,8 @@ "type": "string", "default": "\r\n", "examples": [ - "{\n \"lineTerminator\": \"\\r\\n\"\n}\n", - "{\n \"lineTerminator\": \"\\n\"\n}\n" + "`\r`", + "`\n`" ] }, "nullSequence": { @@ -519,7 +535,7 @@ "description": "Specifies the null sequence, for example, `\\N`.", "type": "string", "examples": [ - "{\n \"nullSequence\": \"\\N\"\n}\n" + "`\\N`" ] }, "quoteChar": { @@ -528,7 +544,7 @@ "type": "string", "default": "\"", "examples": [ - "{\n \"quoteChar\": \"'\"\n}\n" + "`'`" ] }, "escapeChar": { @@ -536,33 +552,27 @@ "description": "Specifies a one-character string to use as the escape character.", "type": "string", "examples": [ - "{\n \"escapeChar\": \"\\\\\"\n}\n" + "`\\`" ] }, "skipInitialSpace": { "title": "Skip Initial Space", "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", "type": "boolean", - "default": false, - "examples": [ - "{\n \"skipInitialSpace\": true\n}\n" - ] + "default": false }, "header": { "title": "Header", "description": "Specifies if the file includes a header row, always as the first row in the file.", "type": "boolean", - "default": true, - "examples": [ - "{\n \"header\": true\n}\n" - ] + "default": true }, "commentChar": { "title": "Comment Character", "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", "type": "string", "examples": [ - "{\n \"commentChar\": \"#\"\n}\n" + "`#`" ] }, "caseSensitiveHeader": { @@ -570,16 +580,12 @@ "description": "Specifies if the case of headers is meaningful.", "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", "type": "boolean", - "default": false, - "examples": [ - "{\n \"caseSensitiveHeader\": true\n}\n" - ] + "default": false } }, "examples": [ - "{\n \"dialect\": {\n \"delimiter\": \";\"\n }\n}\n", - "{\n \"dialect\": {\n \"delimiter\": \"\\t\",\n \"quoteChar\": \"'\",\n \"commentChar\": \"#\"\n }\n}\n" - ] + "{'delimiter': ';' }" + ] }, "format": { "propertyOrder": 80, @@ -588,7 +594,7 @@ "context": "`csv`, `xls`, `json` are examples of common formats.", "type": "string", "examples": [ - "{\n \"format\": \"xls\"\n}\n" + "`xls`" ] }, "mediatype": { @@ -598,7 +604,7 @@ "type": "string", "pattern": "^(.+)/(.+)$", "examples": [ - "{\n \"mediatype\": \"text/csv\"\n}\n" + "`text/csv`" ] }, "encoding": { @@ -608,7 +614,7 @@ "type": "string", "default": "utf-8", "examples": [ - "{\n \"encoding\": \"utf-8\"\n}\n" + "`utf-8`" ] }, "bytes": { @@ -620,7 +626,7 @@ "description": "The size of this resource in bytes.", "type": "integer", "examples": [ - "{\n \"bytes\": 2082\n}\n" + "2082" ] }, "hash": { @@ -633,28 +639,27 @@ "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", "examples": [ - "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", - "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" + "d25c9c77f588f5dc32059d2da1136c02", + "SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0" ] } } - }, - "examples": [ - "{\n \"resources\": [\n {\n \"name\": \"my-data\",\n \"data\": [\n \"data.csv\"\n ],\n \"schema\": \"tableschema.json\",\n \"mediatype\": \"text/csv\"\n }\n ]\n}\n" - ] + } }, "sources": { - "propertyOrder": 110, + "propertyOrder": 140, "options": { "hidden": true }, "title": "Sources", - "description": "Array of data sources formatted as a source. Recommended to be documented either here at the top-level or for each individual resource", + "description": "The raw sources for this data package which describe where the data came from. Can be alternatively specified at the individual resource-level.", "type": "array", "minItems": 0, + "examples": [ + "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" + ], "items": { "title": "Source", - "description": "A source file.", "type": "object", "required": [ "title" @@ -662,37 +667,42 @@ "properties": { "title": { "title": "Title", - "description": "A human-readable title.", + "description": "Human-readable title of the data source.", "type": "string", "examples": [ - "{\n \"title\": \"My Package Title\"\n}\n" + "Trolley CAD/AVL System" ] }, - "path": { - "title": "Path", - "description": "A fully qualified URL, or a POSIX file path.", + "component": { + "title": "Component", + "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", "examples": [ - "{\n \"path\": \"file.csv\"\n}\n", - "{\n \"path\": \"http://example.com/file.csv\"\n}\n" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." + "CAD/AVL", + "AFC", + "APC" + ] }, - "email": { - "title": "Email", - "description": "An email address.", + "product": { + "title": "Product", + "description": "What product was used to generate this data (directly or indirectly)?", "type": "string", - "format": "email", "examples": [ - "{\n \"email\": \"example@example.com\"\n}\n" + "TrolleyMaster" ] + }, + "product_version": { + "title": "Product Version", + "description": "Describe the version of the product was used.", + "type": "string" + }, + "vendor": { + "title": "Vendor", + "description": "What company makes this product?", + "type": "string" } } - }, - "examples": [ - "{\n \"sources\": [\n {\n \"title\": \"World Bank and OECD\",\n \"path\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" - ] + } } } } From ec3c7e825b285e7f5bfd3205650ad3c9a6a68fc5 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Fri, 14 Jul 2023 14:35:40 -0700 Subject: [PATCH 18/21] add local validation scripts --- bin/validate-data-package | 68 +++++++++++++++++++ bin/validate-data-package-json | 64 +++++++++++++++++ docs/datapackage.md | 40 ++++++++++- main.py | 31 ++++++--- mkdocs.yml | 1 + samples/template/{ => TIDES}/datapackage.json | 22 +++--- tests/validate_samples | 8 ++- 7 files changed, 207 insertions(+), 27 deletions(-) create mode 100644 bin/validate-data-package create mode 100644 bin/validate-data-package-json rename samples/template/{ => TIDES}/datapackage.json (91%) diff --git a/bin/validate-data-package b/bin/validate-data-package new file mode 100644 index 00000000..7f95647e --- /dev/null +++ b/bin/validate-data-package @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Script: validate_frictionless_package.sh +# Description: Bash script to validate a Frictionless Data Package using the Frictionless CLI. +# Usage: validate_frictionless_package.sh [-v tides_version | -l local_schema_location] [-d dataset_location] +# -v tides_version: Optional. Specify the version of the TIDES specification or 'local' to +# use a local schema. Default is to use the schema specified in the datapackage. +# -l local_schema_location: Optional. Specify the location of the local schema directory. +# Default is '../spec'. Is only used if tides_version = local. +# -d dataset_location: Optional. Specify the location of the TIDES datapackage.json. +# Default is the current directory. + +# Set default values +tides_version="" +local_schema_location="../spec" +dataset_location="." + +# Parse command-line arguments +while getopts ":v:l:d:" opt; do + case $opt in + v) + tides_version=$OPTARG + ;; + l) + local_schema_location=$OPTARG + ;; + d) + dataset_location=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +# Create a temporary data package if using a different schema reference or a local schema +tmp_datapackage="" +if [ "$tides_version" != "" ] then + tmp_datapackage=$(mktemp) + cp "$dataset_location/datapackage.json" "$tmp_datapackage" +fi + +# Set the schema URL based on the option chosen +schema_url="" +if [ "$tides_version" == "local" ]; then + schema_path_prefix="$local_schema_location" +else + schema_path_prefix="https://raw.githubusercontent.com/TIDES-transit/TIDES/$tides_version/spec" +fi + +# Update the 'schema' property in the temporary copy of the datapackage.json file, if applicable +if [ "$tmp_datapackage" != "" ]; then + schema_file=$(echo "$tmp_datapackage" | sed 's/\//\\\//g') + sed -E -i "s/\"schema\": \"[^\/]+\.schema\.json\"/\"schema\": \"$schema_path_prefix\/\${schema_file##*\/}\"/g" "$tmp_datapackage" + dataset_location="$tmp_datapackage" +fi + +# Validate the data package JSON against the TIDES schema +./validate-data-package-json.sh -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" + +# Validate the Frictionless Data Package using the Frictionless CLI +frictionless validate "$dataset_location" + +# Remove the temporary data package file, if applicable +if [ "$tmp_datapackage" != "" ]; then + rm "$tmp_datapackage" +fi diff --git a/bin/validate-data-package-json b/bin/validate-data-package-json new file mode 100644 index 00000000..504e08ca --- /dev/null +++ b/bin/validate-data-package-json @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Script to validate a local JSON file against a schema specified in a GitHub repository. +# Usage: validate-data-package-json.sh [-r ref | -l local_schema_location] [-f datapackage_file] +# -r ref: Optional. Specify the ref name of the GitHub repository. Default is 'main'. +# -l local_schema_location: Optional. Specify the location of the local schema directory. +# -f datapackage_file: Optional. Specify the location of the datapackage.json file. Default is 'datapackage.json' in the execution directory. + +# Check if jsonschema-cli is installed +command -v jsonschema-cli >/dev/null 2>&1 || { + echo >&2 "jsonschema-cli is required but not found. You can install it using 'pip install jsonschema-cli'. Aborting." + exit 1 +} + +# Set default values +ref="main" +local_schema_location="" +datapackage_file="datapackage.json" + +# Parse command-line arguments +while getopts ":r:l:f:" opt; do + case $opt in + r) + ref=$OPTARG + ;; + l) + local_schema_location=$OPTARG + ;; + f) + datapackage_file=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +echo "Validating data package file in $dataset_location" + +# Set the temporary directory path +temp_dir=$(mktemp -d) + +# Set the schema file path based on the option chosen +schema_file="" +if [ "$local_schema_location" != "" ]; then + schema_file="$local_schema_location/tides-data-package.json" +else + # Download the schema file to the temporary directory + schema_url="https://raw.githubusercontent.com/TIDES-transit/TIDES/$ref/spec/tides-data-package.json" + schema_file="$temp_dir/data-package.json" + + if curl -s --head "$schema_url/tides-data-package.json" >/dev/null; then + echo "Schema file not found on GitHub for the specified TIDES version: $tides_version" + exit 1 + fi +curl -o "$schema_file" "$schema_url" +fi + +# Validate datapackage against the downloaded schema +jsonschema-cli validate "$schema_file" "$datapackage_file" + +# Clean up the temporary directory +rm -rf "$temp_dir" diff --git a/docs/datapackage.md b/docs/datapackage.md index 58a7aa8a..3f04945e 100644 --- a/docs/datapackage.md +++ b/docs/datapackage.md @@ -16,8 +16,44 @@ Required and recommended fields for each `tabluar-data-resource` are as follows: ## Template -A `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/data/template/TIDES/datapackage.json). +The canonical `datapackage.json` template is available at [`/data/template/TIDES/datapackage.json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/samples/template/TIDES/datapackage.json). -Once `datapackage.json` is created for your data, you can easily conduct [data validation](#validation) using a variet of tools. +!!! warning + This version of `tides-data-package` template is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` template if you are viewing the `main` documentation version. + +{{ include_file('samples/template/TIDES/datapackage.json',code_type='json') }} ## Validation + +There are lots of options for validating your `datapackage.json` file including: + +- [Command Line Interface (CLI) Script](#cli) +- [Various online websites](#point-and-drool) + +### CLI + +You can easily validate your data package file with the script provided in [`/bin/validate-data-package-json`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/bin/validate-data-package-json) + +??? tip "installation requirements" + + Make sure you have jsonschema-cli installed. You can install it specifically or with all of the other suggested tools using one of the commands below: + + ```sh + pip install jsonschema-cli + pip install -r requirements.txt + ``` + +```sh title="usage" +validate-data-package-json -f my-datapackage.json +``` + +{{ include_file('bin/validate-data-package-json',code_type='sh') }} + +### Point-and-Drool + +Because a `tides-data-package` is just a json-schema, you can use the myriad of different json-schema validator out there on the web. Use the [canonical `tides-data-package`](https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/tides-data-package.json) or copy and paste the version from below. + +!!! warning + This version of `tides-data-package` is dependent on the version of the documentation you are viewing and only represents the canonical `tides-data-package` if you are viewing the `main` documentation version. + +{{ include_file('spec/tides-data-package.json',code_type='json') }} diff --git a/main.py b/main.py index 8900b862..38d9037b 100644 --- a/main.py +++ b/main.py @@ -30,19 +30,21 @@ def define_env(env): """ @env.macro - def list_samples(data_dir: str) -> str: + def list_samples(sample_dir: str) -> str: """Outputs a simple list of the directories in a folder in markdown. Args: - data_dir (str):directory to search in + sample_dir (str):directory to search in Returns: str: markdown-formatted list """ - + EXCLUDE = ["template"] fields = ["Sample", "Agency", "Resources", "Vendors"] - data_dir = os.path.join(env.project_dir, data_dir) + sample_dir = os.path.join(env.project_dir, sample_dir) samples = [ - d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d)) + d + for d in os.listdir(sample_dir) + if os.path.isdir(os.path.join(sample_dir, d)) and d not in EXCLUDE ] md_table = ( @@ -50,12 +52,16 @@ def list_samples(data_dir: str) -> str: ) for s in samples: - md_table += datapackage_to_row(os.path.join(data_dir, s), fields) + md_table += datapackage_to_row(os.path.join(sample_dir, s, "TIDES"), fields) return md_table @env.macro def include_file( - filename: str, downshift_h1=True, start_line: int = 0, end_line: int = None + filename: str, + downshift_h1=True, + start_line: int = 0, + end_line: int = None, + code_type: str = None, ): """ Include a file, optionally indicating start_line and end_line. @@ -69,6 +75,8 @@ def include_file( start_line (Optional): if included, will start including the file from this line (indexed to 0) end_line (Optional): if included, will stop including at this line (indexed to 0) + code_type: if not None, will encapsulate the resulting file in + ```..file contents...``` """ full_filename = os.path.join(env.project_dir, filename) with open(full_filename, "r") as f: @@ -98,6 +106,9 @@ def include_file( _replace = _replace.replace(_filenamebase, "") content = content.replace(_find, _replace) + + if code_type is not None: + content = f"\n```{code_type} title='{filename}'\n{content}\n```" return content @env.macro @@ -143,7 +154,6 @@ def frictionless_data_package( return "Cannot find sub-schema {sub_schema} in data package file {dp_filename}" if dp["type"] == "array": dp = dp["items"] - log.info(f"dp: \n{dp}") _field_names = [] @@ -153,13 +163,13 @@ def frictionless_data_package( _field_names += dp.get("recommended", []) if include == "all": _field_names = list(dp["properties"].keys()) - log.info(f"Documenting {len(_field_names)} fields") + if not _field_names: return "No fields found to document with parameters." _fields = [] for _f in _field_names: - log.info(f"Documenting: {_f}") + log.debug(f"Documenting: {_f}") _row_entry = {k: v for k, v in dp["properties"][_f].items() if k in INCLUDE} if _f in dp.get("required", []): _row_entry["requirement"] = "required" @@ -184,7 +194,6 @@ def frictionless_data_package( _row_entry["description"] += f"
**Example**:
`{_ex}`" _fields.append(_row_entry) - log.info(f"_fields: {_fields}") dp_df = pd.DataFrame(_fields, columns=INCLUDE) dp_md = dp_df.to_markdown(index=False) diff --git a/mkdocs.yml b/mkdocs.yml index 652be641..a818df60 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,6 +6,7 @@ edit_uri: edit/main/docs theme: name: material features: + - content.code.copy - navigation.tabs - navigation.expand - toc.integrate diff --git a/samples/template/datapackage.json b/samples/template/TIDES/datapackage.json similarity index 91% rename from samples/template/datapackage.json rename to samples/template/TIDES/datapackage.json index 287a659e..0609c7de 100644 --- a/samples/template/datapackage.json +++ b/samples/template/TIDES/datapackage.json @@ -26,7 +26,7 @@ "resources": [ { "name": "devices", - "path": "TIDES/devices.csv", + "path": "devices.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json", "sources": [ { @@ -39,7 +39,7 @@ }, { "name": "vehicle_locations", - "path": "TIDES/vehicle_locations.csv", + "path": "vehicle_locations.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_locations.schema.json", "sources": [ { @@ -52,7 +52,7 @@ }, { "name": "fare_transactions", - "path": "TIDES/fare_transactions.csv", + "path": "fare_transactions.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/fare_transactions.schema.json", "sources": [ { @@ -65,7 +65,7 @@ }, { "name": "train_cars", - "path": "TIDES/train_cars.csv", + "path": "train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", "sources": [ { @@ -78,7 +78,7 @@ }, { "name": "operators", - "path": "TIDES/operators.csv", + "path": "operators.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/operators.schema.json", "sources": [ { @@ -91,7 +91,7 @@ }, { "name": "stop_visits", - "path": "TIDES/stop_visits.csv", + "path": "stop_visits.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/stop_visits.schema.json", "sources": [ { @@ -104,7 +104,7 @@ }, { "name": "vehicle_train_cars", - "path": "TIDES/vehicle_train_cars.csv", + "path": "vehicle_train_cars.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicle_train_cars.schema.json", "sources": [ { @@ -117,7 +117,7 @@ }, { "name": "vehicles", - "path": "TIDES/vehicles.csv", + "path": "vehicles.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/vehicles.schema.json", "sources": [ { @@ -130,7 +130,7 @@ }, { "name": "trips_performed", - "path": "TIDES/trips_performed.csv", + "path": "trips_performed.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/trips_performed.schema.json", "sources": [ { @@ -143,7 +143,7 @@ }, { "name": "station_activities", - "path": "TIDES/station_activities.csv", + "path": "station_activities.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/station_activities.schema.json", "sources": [ { @@ -156,7 +156,7 @@ }, { "name": "passenger_events", - "path": "TIDES/passenger_events.csv", + "path": "passenger_events.csv", "schema": "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/passenger_events.schema.json", "sources": [ { diff --git a/tests/validate_samples b/tests/validate_samples index cac8c226..8f35ba6c 100755 --- a/tests/validate_samples +++ b/tests/validate_samples @@ -1,8 +1,10 @@ #!/usr/bin/env bash echo "Validating sample data" shopt -s nullglob -for file in samples/*/TIDES/datapackage.json; do - echo "Validating: $file" - frictionless validate --schema-sync $file +for dir in samples/*/TIDES; do + echo "Validating: $dir to canonical spec" + ..bin/validate-data-package -d $dir -v main + echo "Validating: $dir to local spec" + ..bin/validate-data-package -d $dir -v local done shopt -u nullglob From 9d8f05f28a91238a46528d04ea3ece0df753b79f Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 20 Sep 2023 10:54:21 -0700 Subject: [PATCH 19/21] pep8/precommit --- .github/workflows/docs.yml | 1 - bin/replace-spec-in-datapackage | 4 ++-- bin/utils | 12 ++++++------ bin/validate-datapackage-to-profile | 8 ++++---- tests/test_samples_to_canonical | 2 +- tests/test_samples_to_local | 2 +- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ad2a0682..96ff4333 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,4 +65,3 @@ jobs: issue-number: ${{ github.event.number }} body: "Documentation available at: [`http://tides-transit.github.io/TIDES/${{ github.head_ref }}`](http://tides-transit.github.io/TIDES/${{ github.head_ref }})" edit-mode: replace - diff --git a/bin/replace-spec-in-datapackage b/bin/replace-spec-in-datapackage index 889800ed..13ee5e1b 100755 --- a/bin/replace-spec-in-datapackage +++ b/bin/replace-spec-in-datapackage @@ -13,7 +13,7 @@ Arguments: " example_usage=" -Example Usage: +Example Usage: bin/replace-spec-in-datapackage samples/template/TIDES spec samples/template/TIDES/datapackage.tmp.json " @@ -95,4 +95,4 @@ jq --arg spec_path_prefix "$spec_path_prefix" --arg profile_file "$profile_file" | .profile = ($profile_file) ' "$output_file" > "$output_file.tmp" && mv "$output_file.tmp" "$output_file" -echo "$output_file" \ No newline at end of file +echo "$output_file" diff --git a/bin/utils b/bin/utils index 75f5a05a..a7fb8235 100755 --- a/bin/utils +++ b/bin/utils @@ -2,9 +2,9 @@ # Check if jsonschema-cli is installed check_jsonschema-cli() { - if ! command -v jsonschema-cli >/dev/null 2>&1; then - echo >&2 "\033[31m!!! jsonschema-cli is required but not found.\033[0m - + if ! command -v jsonschema-cli >/dev/null 2>&1; then + echo >&2 "\033[31m!!! jsonschema-cli is required but not found.\033[0m + You can install it using 'pip install jsonschema-cli'. Aborting." exit 1 fi @@ -12,9 +12,9 @@ check_jsonschema-cli() { # Check if frictionless is installed check_frictionless() { - if ! command -v frictionless >/dev/null 2>&1; then - echo >&2 "\033[31m!!! frictionless is required but not found.\033[0m - + if ! command -v frictionless >/dev/null 2>&1; then + echo >&2 "\033[31m!!! frictionless is required but not found.\033[0m + You can install it using 'pip install frictionless'. Aborting." exit 1 fi diff --git a/bin/validate-datapackage-to-profile b/bin/validate-datapackage-to-profile index 24d92feb..2c02dcf1 100755 --- a/bin/validate-datapackage-to-profile +++ b/bin/validate-datapackage-to-profile @@ -1,16 +1,16 @@ #!/usr/bin/env bash -description="Script to validate a local JSON file against a profile for tides-data-package specified in +description="Script to validate a local JSON file against a profile for tides-data-package specified in profile field or optionally against a remote or local profile." usage=" Usage: validate-datapackage-to-profile [-r remote_spec_ref | -l local_spec_path] [-f datapackage_file] -r remote_spec_ref: Optional. Specify the ref name of the GitHub repository for validating agianst - a remote profile where the profile is in the sub-path /spec/tides-data-package.json. + a remote profile where the profile is in the sub-path /spec/tides-data-package.json. Should not be used with -l option. Example: -r main -l local_spec_path: Optional. Specify the location of the local tides-data-package-json to use. Should not be used with -r option. Example: -l spec - -d dataset_path: Optional. Specify the path of the datapackage.json file. + -d dataset_path: Optional. Specify the path of the datapackage.json file. Default is datapackage.json. Example: -d samples/template/TIDES/datapackage.json " @@ -97,7 +97,7 @@ fi if [ -f "$dataset_path" ]; then datapackage_file="$dataset_path" dataset_path=$(dirname "$dataset_path") -else +else datapackage_file="$dataset_path/datapackage.json" fi check_valid_path "$datapackage_file" diff --git a/tests/test_samples_to_canonical b/tests/test_samples_to_canonical index 0bfc5ac9..cd83d9c9 100755 --- a/tests/test_samples_to_canonical +++ b/tests/test_samples_to_canonical @@ -22,4 +22,4 @@ if output=$("$script" $args 2>&1); then else printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" echo "$output" >&2 -fi \ No newline at end of file +fi diff --git a/tests/test_samples_to_local b/tests/test_samples_to_local index 57041bf8..74286dcd 100755 --- a/tests/test_samples_to_local +++ b/tests/test_samples_to_local @@ -22,4 +22,4 @@ if output=$("$script" $args 2>&1); then else printf "\033[31m!!! %s encountered an error.\033[0m\n" "$script" echo "$output" >&2 -fi \ No newline at end of file +fi From 88e19b0ec26c808f1c5ed18e16d8d6386c4e3e37 Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 20 Sep 2023 10:56:15 -0700 Subject: [PATCH 20/21] came back - re-deleting --- spec/tides-data-package.json | 708 ----------------------------------- 1 file changed, 708 deletions(-) delete mode 100644 spec/tides-data-package.json diff --git a/spec/tides-data-package.json b/spec/tides-data-package.json deleted file mode 100644 index 40078576..00000000 --- a/spec/tides-data-package.json +++ /dev/null @@ -1,708 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TIDES Data Package", - "description": "TIDES Data Package is a simple specification for data access and delivery of tabular TIDES transit data.", - "type": "object", - "required": [ - "title", - "profile", - "resources" - - ], - "recommended":[ - "name", - "description", - "agency", - "ntd_id", - "licenses", - "contributors", - "maintainers" - ], - "properties": { - "profile": { - "const": "tides-data-package", - "propertyOrder": 10, - "title": "Profile", - "description": "The json-schema profile used to validate this datapackage descriptor.", - "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", - "type": "string" - }, - "title": { - "propertyOrder": 20, - "title": "Title", - "description": "A human-readable title.", - "type": "string", - "examples": [ - "Make Believe Trolley 1/1/2001-2/1/2001" - ] - }, - "name": { - "propertyOrder": 30, - "title": "Name", - "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "trolley_2001_1" - ] - }, - "description": { - "propertyOrder": 40, - "format": "textarea", - "title": "Description", - "description": "Short description of TIDES data package.", - "type": "string", - "examples": [ - "Raw and processed data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" - ] - }, - "agency": { - "propertyOrder": 50, - "format": "textarea", - "title": "Agency", - "description": "Transit agency name.", - "type": "string", - "examples": [ - "Make Believe Neighborhood Trolley" - ] - }, - "ntd_id": { - "propertyOrder": 60, - "title": "NTD ID", - "description": "ID for the National Transit Database.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "Background on NTD available at https://www.transit.dot.gov/ntd", - "examples": [ - "0R02-00307" - ] - }, - "contributors": { - "propertyOrder": 70, - "title": "Contributors", - "description": "Array of data contributors.", - "type": "array", - "minItems": 1, - "items": { - "title": "Contributor", - "description": "A contributor to this descriptor.", - "type": "object", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "Daniel Tiger" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "daniel@makebelievetrolley.gov" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this contributor.", - "type": "string", - "example": "Make Believe Neighborhood Trolley" - }, - "role": { - "title": "Role", - "description": "The contributor's role in the project", - "type": "string", - "default": "contributor" - } - }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." - }, - "examples": [ - "{'name':'Daniel Tiger','email': 'daniel@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'contributor'}" - ] - }, - "maintainers": { - "propertyOrder": 80, - "title": "Maintainers", - "description": "Array of data maintainers", - "type": "array", - "minItems": 1, - "items": { - "title": "Maintainer", - "description": "A maintainer of this descriptor.", - "type": "object", - "properties": { - "name": { - "title": "Name", - "description": "A human-readable name.", - "type": "string", - "examples": [ - "Henrietta Pussycat" - ] - }, - "email": { - "title": "Email", - "description": "An email address.", - "type": "string", - "format": "email", - "examples": [ - "henrietta@makebelievetrolley.gov" - ] - }, - "organization": { - "title": "Organization", - "description": "An organizational affiliation for this maintainer.", - "type": "string" - }, - "role": { - "title": "Role", - "description": "The maintainer's role in the project", - "type": "string", - "default": "maintainer" - } - }, - "required": [ - "title" - ], - "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." - }, - "examples": [ - "{'name':'Henrietta Pussycat','email': 'henrietta@makebelievetrolley.gov','organization':'Make Believe Neighborhood Trolley','role':'maintainer'}" - - ] - }, - "licenses": { - "propertyOrder": 90, - "title": "Licenses", - "description": "The license(s) under which this package is published.", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$", - "examples": ["odc-by","CC-BY-4.0"] - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a Unix-style relative file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "https://opendefinition.org/licenses/odc-by/", - "license.md" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title of the license.", - "type": "string", - "examples": [ - "Creative Commons Attribution 4.0" - ] - } - } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" - ] - }, - "resources": { - "propertyOrder": 100, - "title": "Tabular Data Resources", - "description": "Array describing the data files included in your package, formated as a [`tabular-data-resource`](#tabular-data-resource)", - "type": "array", - "minItems": 1, - "items": { - "title": "Tabular Data Resource", - "description": "A Tabular Data Resource.", - "type": "object", - "required": [ - "name", - "path", - "schema", - "profile" - ], - "recommended": [ - "title", - "description", - "sources", - "licenses" - ], - "properties": { - "profile": { - "const": "tabular-data-resource", - "propertyOrder": 10, - "title": "Profile", - "description": "Must be `tabular-data-resource`", - "type": "string", - "examples": [ - "tabular-data-resource" - ] - }, - "name": { - "propertyOrder": 20, - "title": "Name", - "description": "Short, unique [sluggable](https://en.wikipedia.org/wiki/Clean_URL#Slug) identifier string for a data table.", - "type": "string", - "pattern": "^([-a-z0-9._/])+$", - "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", - "examples": [ - "vehicle_locations_2001_01" - ] - }, - "path": { - "propertyOrder": 30, - "title": "Path", - "description": "A reference to the data for this resource, as a valid URI string.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "event/2001/vehicle_locations_2001_01.csv" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "schema": { - "propertyOrder": 40, - "title": "Table Schema", - "description": "An object describing the TIDES Table Schema for this resource.", - "type": "object", - "required": [ - "name", - "path" - ], - "properties": { - "name": { - "propertyOrder": 10, - "title": "Name", - "enum": [ - "devices", - "fare_transactions", - "operators", - "station_activities", - "passenger_events", - "stop_visits", - "train_cars", - "trips_performed", - "vehicle_locations", - "vehicle_train_cars", - "vehicles" - ], - "description": "An identifier string corresponding to the name of one of the table specs in the TIDES data package", - "type": "string", - "examples": [ - "fare_transactions", - "train_cars" - ] - }, - "path": { - "propertyOrder": 20, - "title": "Path", - "description": "A fully qualified public URL, or a relative Unix-style file path, leading to a place where the relevant table spec is defined.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/train_cars.schema.json", - "fare_transactions.schema.json" - ] - } - }, - "examples": [ - "`{'name': 'devices' ,'path': 'https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/devices.schema.json'}`" - ] - }, - "title": { - "title": "Title", - "description": "A human-readable title for the data resource.", - "type": "string", - "examples": [ - "Neighborhood Trolley Vehicle Locations: 1/1/2001-2/1/2001" - ], - "propertyOrder": 50 - }, - "description": { - "title": "Description", - "description": "D", - "type": "string", - "examples": [ - "Vehicle location event data from the Neighborhood of Make Believe Trolley recorded 1/1/2001-2/1/2001" - ], - "propertyOrder": 60, - "format": "textarea" - }, - "homepage": { - "propertyOrder": 70, - "title": "Home Page", - "description": "The home on the web that is related to this data package.", - "type": "string", - "format": "uri", - "examples": [ - "http://example.com" - ] - }, - "sources": { - "propertyOrder": 140, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "The raw sources for this resource which describe where the data came from.", - "type": "array", - "minItems": 0, - "examples": [ - "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" - ], - "items": { - "title": "Source", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "Human-readable title of the data source.", - "type": "string", - "examples": [ - "Trolley CAD/AVL System" - ] - }, - "component": { - "title": "Component", - "description": "The technology component used to generate this data (directly or indirectly)", - "type": "string", - "examples": [ - "CAD/AVL", - "AFC", - "APC" - ] - }, - "product": { - "title": "Product", - "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string", - "examples": [ - "TrolleyMaster" - ] - }, - "product_version": { - "title": "Product Version", - "description": "Describe the version of the product was used.", - "type": "string" - }, - "vendor": { - "title": "Vendor", - "description": "What company makes this product?", - "type": "string" - } - } - } - }, - "licenses": { - "propertyOrder": 90, - "title": "Licenses", - "description": "The license(s) under which this package is published.", - "type": "array", - "minItems": 1, - "items": { - "title": "License", - "description": "An array of license objects which apply to this data package. Must include at least a `name` or `path`", - "type": "object", - "anyOf": [ - { - "required": [ - "name" - ] - }, - { - "required": [ - "path" - ] - } - ], - "properties": { - "name": { - "title": "Open Definition license identifier", - "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", - "type": "string", - "pattern": "^([-a-zA-Z0-9._])+$", - "examples": ["odc-by","CC-BY-4.0"] - }, - "path": { - "title": "Path", - "description": "A fully qualified public URL, or a Unix-style relative file path.", - "type": "string", - "pattern": "^(?=^[^./~])(^((?!\\.{2}).)*$).*$", - "examples": [ - "https://opendefinition.org/licenses/odc-by/", - "license.md" - ], - "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." - }, - "title": { - "title": "Title", - "description": "A human-readable title of the license.", - "type": "string", - "examples": [ - "Creative Commons Attribution 4.0" - ] - } - } - }, - "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", - "examples": [ - "{'name':'CC-BY-4.0','path':'https://opendefinition.org/licenses/cc-by/','name':'Creative Commons Attribution 4.0'}" - ] - }, - "dialect": { - "propertyOrder": 50, - "title": "CSV Dialect", - "description": "The CSV dialect descriptor.", - "type": [ - "string", - "object" - ], - "required": [ - "delimiter", - "doubleQuote" - ], - "properties": { - "csvddfVersion": { - "title": "CSV Dialect schema version", - "description": "A number to indicate the schema version of CSV Dialect. Version 1.0 was named CSV Dialect Description Format and used different field names.", - "type": "number", - "default": 1.2, - "examples:": [ - "1.2" - ] - }, - "delimiter": { - "title": "Delimiter", - "description": "A character sequence to use as the field separator.", - "type": "string", - "default": ",", - "examples": [ - "`;`", - "`|`" - ] - }, - "doubleQuote": { - "title": "Double Quote", - "description": "Specifies the handling of quotes inside fields.", - "context": "If Double Quote is set to true, two consecutive quotes must be interpreted as one.", - "type": "boolean", - "default": true - }, - "lineTerminator": { - "title": "Line Terminator", - "description": "Specifies the character sequence that must be used to terminate rows.", - "type": "string", - "default": "\r\n", - "examples": [ - "`\r`", - "`\n`" - ] - }, - "nullSequence": { - "title": "Null Sequence", - "description": "Specifies the null sequence, for example, `\\N`.", - "type": "string", - "examples": [ - "`\\N`" - ] - }, - "quoteChar": { - "title": "Quote Character", - "description": "Specifies a one-character string to use as the quoting character.", - "type": "string", - "default": "\"", - "examples": [ - "`'`" - ] - }, - "escapeChar": { - "title": "Escape Character", - "description": "Specifies a one-character string to use as the escape character.", - "type": "string", - "examples": [ - "`\\`" - ] - }, - "skipInitialSpace": { - "title": "Skip Initial Space", - "description": "Specifies the interpretation of whitespace immediately following a delimiter. If false, whitespace immediately after a delimiter should be treated as part of the subsequent field.", - "type": "boolean", - "default": false - }, - "header": { - "title": "Header", - "description": "Specifies if the file includes a header row, always as the first row in the file.", - "type": "boolean", - "default": true - }, - "commentChar": { - "title": "Comment Character", - "description": "Specifies that any row beginning with this one-character string, without preceeding whitespace, causes the entire line to be ignored.", - "type": "string", - "examples": [ - "`#`" - ] - }, - "caseSensitiveHeader": { - "title": "Case Sensitive Header", - "description": "Specifies if the case of headers is meaningful.", - "context": "Use of case in source CSV files is not always an intentional decision. For example, should \"CAT\" and \"Cat\" be considered to have the same meaning.", - "type": "boolean", - "default": false - } - }, - "examples": [ - "{'delimiter': ';' }" - ] - }, - "format": { - "propertyOrder": 80, - "title": "Format", - "description": "The file format of this resource.", - "context": "`csv`, `xls`, `json` are examples of common formats.", - "type": "string", - "examples": [ - "`xls`" - ] - }, - "mediatype": { - "propertyOrder": 90, - "title": "Media Type", - "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", - "type": "string", - "pattern": "^(.+)/(.+)$", - "examples": [ - "`text/csv`" - ] - }, - "encoding": { - "propertyOrder": 100, - "title": "Encoding", - "description": "The file encoding of this resource.", - "type": "string", - "default": "utf-8", - "examples": [ - "`utf-8`" - ] - }, - "bytes": { - "propertyOrder": 110, - "options": { - "hidden": true - }, - "title": "Bytes", - "description": "The size of this resource in bytes.", - "type": "integer", - "examples": [ - "2082" - ] - }, - "hash": { - "propertyOrder": 120, - "options": { - "hidden": true - }, - "title": "Hash", - "type": "string", - "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", - "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", - "examples": [ - "d25c9c77f588f5dc32059d2da1136c02", - "SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0" - ] - } - } - } - }, - "sources": { - "propertyOrder": 140, - "options": { - "hidden": true - }, - "title": "Sources", - "description": "The raw sources for this data package which describe where the data came from. Can be alternatively specified at the individual resource-level.", - "type": "array", - "minItems": 0, - "examples": [ - "[ {'title':'Trolley CAD/AVL System','component':'CAD/AVL','product':'TrolleyMaster','product_version':'3.1.4'} ]" - ], - "items": { - "title": "Source", - "type": "object", - "required": [ - "title" - ], - "properties": { - "title": { - "title": "Title", - "description": "Human-readable title of the data source.", - "type": "string", - "examples": [ - "Trolley CAD/AVL System" - ] - }, - "component": { - "title": "Component", - "description": "What technology component was used to generate this data (directly or indirectly)? Examples include `AVL`, `APC`, `AFC`, etc.", - "type": "string", - "examples": [ - "CAD/AVL", - "AFC", - "APC" - ] - }, - "product": { - "title": "Product", - "description": "What product was used to generate this data (directly or indirectly)?", - "type": "string", - "examples": [ - "TrolleyMaster" - ] - }, - "product_version": { - "title": "Product Version", - "description": "Describe the version of the product was used.", - "type": "string" - }, - "vendor": { - "title": "Vendor", - "description": "What company makes this product?", - "type": "string" - } - } - } - } - } -} From cb6dd18a4cbc9686d57277fa957277ab164ae31e Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Mon, 4 Dec 2023 12:48:35 -0800 Subject: [PATCH 21/21] bug/typo fixes --- README.md | 4 ++-- bin/validate-data-package | 10 +++++----- bin/validate-data-package-json | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4f77c820..743589e0 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ bin/validate-datapackage [-v remote_spec_ref | -l local_spec_path] [-d dataset_p Specific files can be validated by running the frictionless framework against them and their corresponding schemas as follows: ```sh -frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json +frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TIDES-transit/TIDES/main/spec/schema.vehicles.json --schema-sync ``` ## Sample Data @@ -48,7 +48,7 @@ frictionless validate vehicles.csv --schema https://raw.githubusercontent.com/TI ### Template -Templates of `datapackage.json` and each TIDES file type are located in the `/samples/template` directory. +Templates of `datapackage.json` and each TIDES file type are located in the `/samples/template` directory. They can be used to build out TIDES data, particuarly samples. Most TIDES data in practice will be directly produced as an output from software or scripts. ## Contributing to TIDES diff --git a/bin/validate-data-package b/bin/validate-data-package index 7f95647e..be74095b 100644 --- a/bin/validate-data-package +++ b/bin/validate-data-package @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Script: validate_frictionless_package.sh +# Script: validate_data_package # Description: Bash script to validate a Frictionless Data Package using the Frictionless CLI. -# Usage: validate_frictionless_package.sh [-v tides_version | -l local_schema_location] [-d dataset_location] +# Usage: validate_data_package [-v tides_version | -l local_schema_location] [-d dataset_location] # -v tides_version: Optional. Specify the version of the TIDES specification or 'local' to # use a local schema. Default is to use the schema specified in the datapackage. # -l local_schema_location: Optional. Specify the location of the local schema directory. @@ -36,7 +36,7 @@ done # Create a temporary data package if using a different schema reference or a local schema tmp_datapackage="" -if [ "$tides_version" != "" ] then +if [ "$tides_version" != "" ]; then tmp_datapackage=$(mktemp) cp "$dataset_location/datapackage.json" "$tmp_datapackage" fi @@ -57,10 +57,10 @@ if [ "$tmp_datapackage" != "" ]; then fi # Validate the data package JSON against the TIDES schema -./validate-data-package-json.sh -v "$tides_version" -d "$dataset_location" -l "$local_schema_location" +./validate-data-package-json -v "$tides_version" -f "$dataset_location" -l "$local_schema_location" # Validate the Frictionless Data Package using the Frictionless CLI -frictionless validate "$dataset_location" +frictionless validate "$dataset_location" --schema-sync # Remove the temporary data package file, if applicable if [ "$tmp_datapackage" != "" ]; then diff --git a/bin/validate-data-package-json b/bin/validate-data-package-json index 504e08ca..4d8495fb 100644 --- a/bin/validate-data-package-json +++ b/bin/validate-data-package-json @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Script to validate a local JSON file against a schema specified in a GitHub repository. -# Usage: validate-data-package-json.sh [-r ref | -l local_schema_location] [-f datapackage_file] +# Usage: validate-data-package-json [-r ref | -l local_schema_location] [-f datapackage_file] # -r ref: Optional. Specify the ref name of the GitHub repository. Default is 'main'. # -l local_schema_location: Optional. Specify the location of the local schema directory. # -f datapackage_file: Optional. Specify the location of the datapackage.json file. Default is 'datapackage.json' in the execution directory.