diff --git a/.github/workflows/edgetest.yml b/.github/workflows/edgetest.yml index ebf0f40d..40489d80 100644 --- a/.github/workflows/edgetest.yml +++ b/.github/workflows/edgetest.yml @@ -14,19 +14,19 @@ jobs: runs-on: ubuntu-latest name: running edgetest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: ref: ${{ github.ref_name }} - name: Set up Python 3.12 - uses: conda-incubator/setup-miniconda@v3 + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 with: auto-update-conda: true python-version: '3.12' channels: conda-forge - name: Setup Java JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: java-version: '17' distribution: 'adopt' @@ -44,7 +44,7 @@ jobs: edgetest -c pyproject.toml --export - name: Create Pull Request - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: branch: edgetest-patch base: ${{ github.ref_name }} diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 32af2bcd..68a856dc 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -12,12 +12,12 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: # fetch all tags so `versioneer` can properly determine current version fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.12' - name: Install dependencies @@ -26,7 +26,7 @@ jobs: run: make sphinx shell: bash - name: Publish - uses: JamesIves/github-pages-deploy-action@4.0.0 + uses: JamesIves/github-pages-deploy-action@d92aa235d04922e8f08b40ce78cc5442fcfbfa2f # v4.8.0 with: branch: gh-pages folder: docs/build/html diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 0797be50..88a9616f 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -13,12 +13,12 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # pin@v2.7.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: # fetch all tags so `versioneer` can properly determine current version fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # pin@v2.3.4 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.12' - name: Install dependencies diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index c71a246d..f34c8e10 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -17,9 +17,9 @@ jobs: lint-and-format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: "3.12" - name: Install dependencies @@ -47,13 +47,13 @@ jobs: PANDAS_VERSION: ${{ matrix.pandas-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} - name: Setup Java JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: java-version: "17" distribution: "adopt" @@ -92,13 +92,13 @@ jobs: PANDAS_VERSION: ${{ matrix.pandas-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} - name: Setup Java JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: java-version: "17" distribution: "adopt" @@ -129,9 +129,9 @@ jobs: PYTHON_VERSION: ${{ matrix.python-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} - name: Install datacompy diff --git a/CLAUDE.md b/CLAUDE.md index 9266d8dd..e1644bb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,6 +80,16 @@ Tolerances (`abs_tol`, `rel_tol`) can be a single float (applied globally) or a - **Imports**: Only absolute imports (relative imports banned via ruff TID252) - **Pre-commit hooks**: ruff (lint + format), trailing whitespace, debug statements, end-of-file fixer, pyproject-fmt +## Testing Conventions + +- Write plain pytest functions, not class-based test suites. Use `def test_*()` at module level. +- Do not group tests into `class Test*` unless the upstream codebase already does so in the same file. + +## Documentation Conventions + +- Do not use em dashes ("--" or "---") in documentation or docstrings; rewrite the sentence instead. +- Do not use emojis in documentation, docstrings, or commit messages. + ## Branching - `develop` is the active development branch for v1 diff --git a/datacompy/cli/__init__.py b/datacompy/cli/__init__.py new file mode 100644 index 00000000..b325e7e3 --- /dev/null +++ b/datacompy/cli/__init__.py @@ -0,0 +1,45 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DataComPy command-line interface. + +Entry point: ``datacompy`` (installed via ``[project.scripts]``) or +``python -m datacompy``. + +Examples +-------- +Compare two CSV files using the Polars backend (default): + +.. code-block:: bash + + datacompy compare --left a.csv --right b.csv --on id + +Emit a JSON report to stdout: + +.. code-block:: bash + + datacompy compare --left a.csv --right b.csv --on id --json + +Use Pandas and compare on the DataFrame index: + +.. code-block:: bash + + datacompy compare --left a.csv --right b.csv --on-index --backend pandas +""" + +from datacompy.cli.main import main +from datacompy.cli.parser import build_parser + +__all__ = ["build_parser", "main"] diff --git a/datacompy/cli/__main__.py b/datacompy/cli/__main__.py new file mode 100644 index 00000000..503c1139 --- /dev/null +++ b/datacompy/cli/__main__.py @@ -0,0 +1,20 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allows ``python -m datacompy`` to invoke the CLI.""" + +from datacompy.cli import main + +raise SystemExit(main()) diff --git a/datacompy/cli/backends.py b/datacompy/cli/backends.py new file mode 100644 index 00000000..57154cb0 --- /dev/null +++ b/datacompy/cli/backends.py @@ -0,0 +1,204 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Backend factory functions. + +Each ``make_*_compare`` function accepts a typed ``CompareArgs`` and the +already-loaded DataFrames, then constructs and returns the appropriate +backend-specific ``Compare`` instance. The factories centralise the +constructor-signature differences between backends (e.g. Pandas supports +``on_index``; Snowflake has no ``cast_column_names_lower``). +""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from datacompy.cli.loaders import is_snowflake_ref + + +@dataclass(frozen=True) +class CompareArgs: + """Typed mirror of the ``compare`` subcommand arguments.""" + + left: str + right: str + format: str | None + on: list[str] | None + on_index: bool + backend: str + abs_tol: float + rel_tol: float + ignore_spaces: bool + ignore_case: bool + ignore_extra_columns: bool + ignore_unique_rows: bool + cast_column_names_lower: bool + csv_delimiter: str + df1_name: str + df2_name: str + sample_count: int + column_count: int + max_unequal_rows: int | None + json: bool + quiet: bool + spark_app_name: str + snowflake_config: Path | None + + +def _default_name(ref: str) -> str: + """Derive a human-readable dataset label from a file path or Snowflake table ref. + + For file paths ``Path.stem`` is used (``"sales_data.parquet"`` → ``"sales_data"``). + For Snowflake table refs the table name (last segment) is used so that + ``"PROD.ANALYTICS.SALES_FACT"`` → ``"SALES_FACT"`` rather than the + misleading ``"PROD.ANALYTICS"`` that ``Path.stem`` would produce. + """ + if is_snowflake_ref(ref): + return ref.rsplit(".", 1)[-1] + return Path(ref).stem + + +def to_compare_args(ns: Any) -> CompareArgs: + """Convert an :class:`argparse.Namespace` to a typed :class:`CompareArgs`.""" + return CompareArgs( + left=ns.left, + right=ns.right, + format=ns.format, + on=ns.on, + on_index=ns.on_index, + backend=ns.backend, + abs_tol=ns.abs_tol, + rel_tol=ns.rel_tol, + ignore_spaces=ns.ignore_spaces, + ignore_case=ns.ignore_case, + ignore_extra_columns=ns.ignore_extra_columns, + ignore_unique_rows=ns.ignore_unique_rows, + cast_column_names_lower=ns.cast_column_names_lower, + csv_delimiter=ns.csv_delimiter, + df1_name=ns.df1_name if ns.df1_name is not None else _default_name(ns.left), + df2_name=ns.df2_name if ns.df2_name is not None else _default_name(ns.right), + sample_count=ns.sample_count, + column_count=ns.column_count, + max_unequal_rows=ns.max_unequal_rows, + json=ns.json, + quiet=ns.quiet, + spark_app_name=ns.spark_app_name, + snowflake_config=ns.snowflake_config, + ) + + +def make_pandas_compare(args: CompareArgs, df1: Any, df2: Any) -> Any: + """Construct a :class:`~datacompy.pandas.PandasCompare`.""" + from datacompy.pandas import PandasCompare + + if args.on_index: + return PandasCompare( + df1, + df2, + on_index=True, + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + df1_name=args.df1_name, + df2_name=args.df2_name, + ignore_spaces=args.ignore_spaces, + ignore_case=args.ignore_case, + cast_column_names_lower=args.cast_column_names_lower, + ) + return PandasCompare( + df1, + df2, + join_columns=args.on, + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + df1_name=args.df1_name, + df2_name=args.df2_name, + ignore_spaces=args.ignore_spaces, + ignore_case=args.ignore_case, + cast_column_names_lower=args.cast_column_names_lower, + ) + + +def make_polars_compare(args: CompareArgs, df1: Any, df2: Any) -> Any: + """Construct a :class:`~datacompy.polars.PolarsCompare`.""" + from datacompy.polars import PolarsCompare + + return PolarsCompare( + df1, + df2, + join_columns=args.on or [], + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + df1_name=args.df1_name, + df2_name=args.df2_name, + ignore_spaces=args.ignore_spaces, + ignore_case=args.ignore_case, + cast_column_names_lower=args.cast_column_names_lower, + ) + + +def make_spark_compare(args: CompareArgs, spark: Any, df1: Any, df2: Any) -> Any: + """Construct a :class:`~datacompy.spark.SparkSQLCompare`.""" + try: + from datacompy.spark import SparkSQLCompare + except ImportError as exc: + from datacompy.cli.errors import MissingExtraError + + raise MissingExtraError( + "Spark backend requires 'datacompy[spark]'. " + "Install it with: pip install datacompy[spark]" + ) from exc + + return SparkSQLCompare( + spark, + df1, + df2, + join_columns=args.on or [], + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + df1_name=args.df1_name, + df2_name=args.df2_name, + ignore_spaces=args.ignore_spaces, + ignore_case=args.ignore_case, + cast_column_names_lower=args.cast_column_names_lower, + ) + + +def make_snowflake_compare( + args: CompareArgs, session: Any, ref1: Any, ref2: Any +) -> Any: + """Construct a :class:`~datacompy.snowflake.SnowflakeCompare`.""" + try: + from datacompy.snowflake import SnowflakeCompare + except ImportError as exc: + from datacompy.cli.errors import MissingExtraError + + raise MissingExtraError( + "Snowflake backend requires 'datacompy[snowflake]'. " + "Install it with: pip install datacompy[snowflake]" + ) from exc + + return SnowflakeCompare( + session, + ref1, + ref2, + join_columns=args.on, + abs_tol=args.abs_tol, + rel_tol=args.rel_tol, + df1_name=args.df1_name, + df2_name=args.df2_name, + ignore_spaces=args.ignore_spaces, + ignore_case=args.ignore_case, + ) diff --git a/datacompy/cli/compare.py b/datacompy/cli/compare.py new file mode 100644 index 00000000..86e2ba0b --- /dev/null +++ b/datacompy/cli/compare.py @@ -0,0 +1,169 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the ``datacompy compare`` subcommand.""" + +import argparse +from contextlib import ExitStack + +from datacompy.base import BaseCompare +from datacompy.cli.backends import ( + CompareArgs, + make_pandas_compare, + make_polars_compare, + make_snowflake_compare, + make_spark_compare, + to_compare_args, +) +from datacompy.cli.errors import BadArgsError +from datacompy.cli.loaders import ( + infer_format, + load_pandas, + load_polars, + load_snowflake, + load_spark, +) +from datacompy.cli.output import emit +from datacompy.report import ReportData + + +def run_compare(ns: argparse.Namespace) -> int: + """Execute a comparison and return the appropriate exit code. + + Parameters + ---------- + ns: + Parsed :class:`argparse.Namespace` from the ``compare`` subparser. + + Returns + ------- + int + ``0`` -- datasets match (within any specified thresholds). + ``1`` -- datasets differ or a threshold was violated. + Raises :class:`~datacompy.cli.errors.CLIError` (exit ``2``) on + invalid arguments, missing files, or unsupported backends. + """ + args = to_compare_args(ns) + _validate_arg_combinations(args) + + with ExitStack() as stack: + compare = _build_backend(args, stack) + report_data = compare.build_report_data( + sample_count=args.sample_count, + column_count=args.column_count, + ) + emit(report_data, as_json=args.json, quiet=args.quiet) + return 0 if _matched(args, report_data, compare) else 1 + + +def _validate_arg_combinations(args: CompareArgs) -> None: + """Raise :class:`~datacompy.cli.errors.BadArgsError` on invalid argument combinations. + + Single-value constraints (e.g. delimiter length, non-negative counts) are + enforced earlier by argparse ``type=`` callables in ``parser.py``. This + function handles rules that require comparing two or more arguments. + """ + if args.on_index and args.backend != "pandas": + raise BadArgsError("--on-index is only supported with --backend pandas.") + if args.on_index and args.on: + raise BadArgsError("--on and --on-index are mutually exclusive.") + if not args.on_index and not args.on: + raise BadArgsError( + "--on is required (or --on-index for the pandas backend). " + "Specify at least one join column with --on COL." + ) + if args.ignore_unique_rows and args.max_unequal_rows is None: + raise BadArgsError( + "--ignore-unique-rows requires --max-unequal-rows to be set." + ) + + +def _build_backend(args: CompareArgs, stack: ExitStack) -> BaseCompare: + """Load both datasets and return the appropriate ``*Compare`` instance. + + *stack* is an :class:`~contextlib.ExitStack` that owns session cleanup for + Spark and Snowflake. Registering sessions on the stack ensures they are + closed even when loading or comparison raises an exception. + """ + if args.backend == "pandas": + return _build_pandas(args) + elif args.backend == "polars": + return _build_polars(args) + elif args.backend == "spark": + return _build_spark(args, stack) + elif args.backend == "snowflake": + return _build_snowflake(args, stack) + else: + raise BadArgsError( + f"Unknown backend: {args.backend!r}" + ) # unreachable — argparse choices= enforces the valid set + + +def _build_pandas(args: CompareArgs) -> BaseCompare: + fmt_l = infer_format(args.left, args.format) + fmt_r = infer_format(args.right, args.format) + df1 = load_pandas(args.left, fmt_l, csv_delimiter=args.csv_delimiter) + df2 = load_pandas(args.right, fmt_r, csv_delimiter=args.csv_delimiter) + return make_pandas_compare(args, df1, df2) + + +def _build_polars(args: CompareArgs) -> BaseCompare: + fmt_l = infer_format(args.left, args.format) + fmt_r = infer_format(args.right, args.format) + df1 = load_polars(args.left, fmt_l, csv_delimiter=args.csv_delimiter) + df2 = load_polars(args.right, fmt_r, csv_delimiter=args.csv_delimiter) + return make_polars_compare(args, df1, df2) + + +def _build_spark(args: CompareArgs, stack: ExitStack) -> BaseCompare: + from datacompy.cli.sessions import get_spark_session + + spark = get_spark_session(args.spark_app_name) + stack.callback(spark.stop) + fmt_l = infer_format(args.left, args.format) + fmt_r = infer_format(args.right, args.format) + df1 = load_spark(spark, args.left, fmt_l, csv_delimiter=args.csv_delimiter) + df2 = load_spark(spark, args.right, fmt_r, csv_delimiter=args.csv_delimiter) + return make_spark_compare(args, spark, df1, df2) # type: ignore[no-any-return] + + +def _build_snowflake(args: CompareArgs, stack: ExitStack) -> BaseCompare: + from datacompy.cli.sessions import get_snowflake_session + + session = get_snowflake_session(args.snowflake_config) + stack.callback(session.close) + ref1 = load_snowflake( + session, args.left, args.format, csv_delimiter=args.csv_delimiter + ) + ref2 = load_snowflake( + session, args.right, args.format, csv_delimiter=args.csv_delimiter + ) + return make_snowflake_compare(args, session, ref1, ref2) # type: ignore[no-any-return] + + +def _matched(args: CompareArgs, report_data: ReportData, compare: BaseCompare) -> bool: + """Return ``True`` when the comparison satisfies the configured threshold.""" + if args.max_unequal_rows is not None: + total_diff = report_data.row_summary.unequal_rows + if not args.ignore_unique_rows: + total_diff += ( + report_data.row_summary.df1_unique + report_data.row_summary.df2_unique + ) + column_ok = args.ignore_extra_columns or ( + report_data.column_summary.df1_unique == 0 + and report_data.column_summary.df2_unique == 0 + ) + return column_ok and total_diff <= args.max_unequal_rows + return compare.matches(ignore_extra_columns=args.ignore_extra_columns) diff --git a/datacompy/cli/errors.py b/datacompy/cli/errors.py new file mode 100644 index 00000000..76aa5b16 --- /dev/null +++ b/datacompy/cli/errors.py @@ -0,0 +1,38 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI-specific exception hierarchy. + +All exceptions here carry an ``exit_code`` of ``2`` so that ``main()`` +can catch them at a single site and return the right exit code. +""" + + +class CLIError(Exception): + """Base class for all CLI errors. Always maps to exit code 2.""" + + exit_code: int = 2 + + +class BadArgsError(CLIError): + """Raised when the user provides logically invalid argument combinations.""" + + +class LoadError(CLIError): + """Raised when a source file cannot be read.""" + + +class MissingExtraError(CLIError): + """Raised when a required optional dependency is not installed.""" diff --git a/datacompy/cli/loaders.py b/datacompy/cli/loaders.py new file mode 100644 index 00000000..ae906b3b --- /dev/null +++ b/datacompy/cli/loaders.py @@ -0,0 +1,337 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""File loaders for each supported backend. + +Each ``load_*`` function accepts a path (or URI) string and a format +string (``"csv"``, ``"parquet"``, or ``"json"``) and returns a +backend-appropriate DataFrame. + +Cloud URIs (``s3://``, ``gs://``, ``abfs://``) are passed through +as-is; they work when the user has installed the relevant optional +filesystem library (``s3fs``, ``gcsfs``, ``adlfs``). +""" + +import re +from pathlib import Path +from typing import Any + +import pandas as pd +import polars as pl + +from datacompy.cli.errors import BadArgsError, LoadError, MissingExtraError + +_TABLE_REF_RE = re.compile(r"^[a-zA-Z_$][\w$]*(\.[a-zA-Z_$][\w$]*){1,2}$") + +_NON_TABLE_REF_EXTENSIONS = frozenset( + { + ".csv", + ".tsv", + ".parquet", + ".pq", + ".json", + ".jsonl", + ".ndjson", + ".txt", + ".gz", + ".zip", + } +) + + +def is_snowflake_ref(ref: str) -> bool: + """Return ``True`` when *ref* looks like a Snowflake ``[db.]schema.table`` identifier. + + A ref is considered a table identifier when it: + - contains no path separators (rules out file paths and URIs), + - does not end with a recognised file-like extension (rules out + ``data.csv``, ``archive.zip``, ``snapshot.parquet.gz``, etc.), and + - matches the ``word.word[.word]`` pattern (2- or 3-part dotted identifier). + + Note: ``.gz`` and ``.zip`` are included in the extension guard so that + compressed file paths are never mistaken for table refs. They are not + loadable by the CLI directly — use ``--format csv`` (etc.) with the + underlying reader if it supports the compression format. + """ + if "/" in ref or "\\" in ref: + return False + if Path(ref).suffix.lower() in _NON_TABLE_REF_EXTENSIONS: + return False + return bool(_TABLE_REF_RE.match(ref)) + + +_FORMAT_MAP: dict[str, list[str]] = { + "csv": [".csv", ".tsv"], + "parquet": [".parquet", ".pq"], + "json": [".json", ".jsonl", ".ndjson"], +} + + +def infer_format(path: str, override: str | None) -> str: + """Return the file format string for *path*. + + Parameters + ---------- + path: + File path or URI. + override: + Explicit format string from ``--format``. When provided it is + returned directly without inspecting *path*. + + Returns + ------- + str + One of ``"csv"``, ``"parquet"``, or ``"json"``. + + Raises + ------ + BadArgsError + When the extension is not recognised and *override* is ``None``. + """ + if override is not None: + return override + ext = Path(path).suffix.lower() + for fmt, exts in _FORMAT_MAP.items(): + if ext in exts: + return fmt + raise BadArgsError( + f"Cannot infer format from extension {ext!r}. " + "Use --format csv|parquet|json to specify it explicitly." + ) + + +def load_pandas(path: str, fmt: str, csv_delimiter: str = ",") -> pd.DataFrame: + """Load *path* into a :class:`pandas.DataFrame`. + + Parameters + ---------- + path: + Local path or cloud URI. + fmt: + One of ``"csv"``, ``"parquet"``, ``"json"``. + csv_delimiter: + Field delimiter used when *fmt* is ``"csv"`` (default: comma). + + Raises + ------ + LoadError + On any I/O or parse error (``FileNotFoundError``, ``OSError``, + corrupt file, etc.). + BadArgsError + On unsupported *fmt*. + """ + try: + if fmt == "csv": + return pd.read_csv(path, sep=csv_delimiter) + if fmt == "parquet": + return pd.read_parquet(path) + if fmt == "json": + return pd.read_json(path) + except FileNotFoundError as exc: + raise LoadError(f"File not found: {path}") from exc + except Exception as exc: + raise LoadError(f"Cannot read {path}: {exc}") from exc + raise BadArgsError(f"Unsupported format for pandas loader: {fmt!r}") + + +def load_polars(path: str, fmt: str, csv_delimiter: str = ",") -> pl.DataFrame: + """Load *path* into a :class:`polars.DataFrame`. + + Parameters + ---------- + path: + Local path or cloud URI. + fmt: + One of ``"csv"``, ``"parquet"``, ``"json"``. + csv_delimiter: + Field delimiter used when *fmt* is ``"csv"`` (default: comma). + + Raises + ------ + LoadError + On file-not-found or I/O errors. + BadArgsError + On unsupported *fmt*. + """ + try: + if fmt == "csv": + return pl.read_csv(path, separator=csv_delimiter) + if fmt == "parquet": + return pl.read_parquet(path) + if fmt == "json": + return pl.read_json(path) + except FileNotFoundError as exc: + raise LoadError(f"File not found: {path}") from exc + except Exception as exc: + raise LoadError(f"Cannot read {path}: {exc}") from exc + raise BadArgsError(f"Unsupported format for polars loader: {fmt!r}") + + +def load_spark(spark: Any, path: str, fmt: str, csv_delimiter: str = ",") -> Any: + """Load *path* into a PySpark DataFrame. + + Parameters + ---------- + spark: + A live :class:`pyspark.sql.SparkSession`. + path: + Local path or cloud URI supported by the active Hadoop connectors. + fmt: + One of ``"csv"``, ``"parquet"``, ``"json"``. + csv_delimiter: + Field delimiter used when *fmt* is ``"csv"`` (default: comma). + + Raises + ------ + LoadError + On I/O errors reported by Spark. + BadArgsError + On unsupported *fmt*. + """ + try: + if fmt == "csv": + return spark.read.csv( + path, header=True, inferSchema=True, sep=csv_delimiter + ) + if fmt == "parquet": + return spark.read.parquet(path) + if fmt == "json": + return spark.read.json(path) + except Exception as exc: + raise LoadError(f"Spark cannot read {path}: {exc}") from exc + raise BadArgsError(f"Unsupported format for Spark loader: {fmt!r}") + + +def _expand_table_ref(session: Any, ref: str) -> str: + """Ensure *ref* is a fully-qualified ``db.schema.table`` identifier. + + A 3-part ref is returned unchanged. A 2-part ``schema.table`` ref is + expanded by prepending the session's current database. + + Raises + ------ + BadArgsError + When *ref* is 2-part and the session has no current database. Users + can fix this by passing the fully-qualified ``db.schema.table`` form + or by setting ``SNOWFLAKE_DATABASE`` in their environment. + """ + parts = ref.split(".") + if len(parts) == 3: + return ref + db = session.get_current_database() + if not db: + raise BadArgsError( + f"Cannot resolve {ref!r} to a fully-qualified table name: the " + "Snowflake session has no current database. Either use the " + "db.schema.table form or set SNOWFLAKE_DATABASE." + ) + return f"{db}.{ref}" + + +def load_snowflake( + session: Any, ref: str, fmt: str | None, csv_delimiter: str = "," +) -> Any: + """Load *ref* for use with :class:`~datacompy.snowflake.SnowflakeCompare`. + + When *ref* looks like ``db.schema.table`` (3-part) or ``schema.table`` + (2-part) it is resolved to a fully-qualified table name and returned + directly. Otherwise *ref* is treated as a local file, loaded via + Pandas, and staged to a temporary Snowflake table whose name is returned. + + Parameters + ---------- + session: + A live :class:`snowflake.snowpark.Session`. + ref: + A ``db.schema.table`` (3-part) or ``schema.table`` (2-part) identifier, + or a local file path. 2-part refs are expanded using the session's + current database. + fmt: + File format; only used when *ref* is a local file. + csv_delimiter: + Field delimiter used when *fmt* is ``"csv"`` (default: comma). + + Returns + ------- + str + A ``db.schema.table`` string usable by + :class:`~datacompy.snowflake.SnowflakeCompare`. + + Raises + ------ + MissingExtraError + When ``snowflake.snowpark`` is not installed. + BadArgsError + When a ref cannot be resolved or a local file cannot be staged. + LoadError + On file read or Snowflake staging errors. + """ + if is_snowflake_ref(ref): + return _expand_table_ref(session, ref) + return _stage_file_to_snowflake(session, ref, fmt, csv_delimiter) + + +def _stage_file_to_snowflake( + session: Any, path: str, fmt: str | None, csv_delimiter: str = "," +) -> str: + """Load a local file via Pandas and write it to a temporary Snowflake table. + + Returns the fully-qualified ``db.schema.table`` name of the temp table. + """ + try: + import snowflake.snowpark # noqa: F401 + except ImportError as exc: + raise MissingExtraError( + "Snowflake backend requires 'datacompy[snowflake]'. " + "Install it with: pip install datacompy[snowflake]" + ) from exc + + actual_fmt = fmt or infer_format(path, None) + if actual_fmt not in ("csv", "parquet", "json"): + raise BadArgsError(f"Unsupported format for Snowflake loader: {actual_fmt!r}") + + df_pd = load_pandas(path, actual_fmt, csv_delimiter=csv_delimiter) + + from uuid import uuid4 + + db = session.get_current_database() + schema = session.get_current_schema() + if not db or not schema: + missing_vars = ", ".join( + v + for v, val in [("SNOWFLAKE_DATABASE", db), ("SNOWFLAKE_SCHEMA", schema)] + if not val + ) + raise BadArgsError( + f"Cannot stage {path!r} to Snowflake: session has no current " + f"database/schema. Set {missing_vars} or use the db.schema.table " + "form directly." + ) + + table_name = f"DATACOMPY_TMP_{uuid4().hex[:8].upper()}" + try: + session.write_pandas( + df_pd, + table_name=table_name, + auto_create_table=True, + table_type="temp", + overwrite=True, + ) + except Exception as exc: + raise LoadError( + f"Failed to stage {path} to Snowflake temp table: {exc}" + ) from exc + + return f"{db}.{schema}.{table_name}" diff --git a/datacompy/cli/main.py b/datacompy/cli/main.py new file mode 100644 index 00000000..07243822 --- /dev/null +++ b/datacompy/cli/main.py @@ -0,0 +1,61 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entry point for the ``datacompy`` CLI. + +``main()`` is the function registered as ``[project.scripts] datacompy`` +in ``pyproject.toml`` and called by ``datacompy/cli/__main__.py`` for +``python -m datacompy`` invocations. +""" + +from collections.abc import Sequence + +from datacompy.cli.errors import CLIError +from datacompy.cli.output import print_error +from datacompy.cli.parser import build_parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Parse *argv* and dispatch the appropriate subcommand. + + Parameters + ---------- + argv: + Argument list. When ``None`` :data:`sys.argv` is used by argparse. + + Returns + ------- + int + Exit code: ``0`` match, ``1`` mismatch, ``2`` error. + Argparse itself calls ``sys.exit(2)`` on parse failures, which + propagates before this function returns. + """ + parser = build_parser() + args = parser.parse_args(list(argv) if argv is not None else None) + try: + result: int = args.func(args) + return result + except CLIError as exc: + if args.debug: + raise + print_error(str(exc)) + return exc.exit_code + except FileNotFoundError as exc: + if args.debug: + raise + print_error(f"file not found: {exc.filename or 'unknown'}") + return 2 + # All other exceptions are unexpected bugs — let them propagate as tracebacks. + # Run with --debug to see the full traceback for any error. diff --git a/datacompy/cli/output.py b/datacompy/cli/output.py new file mode 100644 index 00000000..dcb18758 --- /dev/null +++ b/datacompy/cli/output.py @@ -0,0 +1,60 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Output helpers: emit a :class:`~datacompy.report.ReportData` to stdout.""" + +import json as _json +import sys +from typing import Any + +import numpy as np + +from datacompy.report import ReportData + + +def _json_default(obj: Any) -> Any: + """Handle numpy scalar types that the stdlib JSON encoder can't serialise.""" + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + if isinstance(obj, np.bool_): + return bool(obj) + return str(obj) + + +def emit(report_data: ReportData, *, as_json: bool, quiet: bool) -> None: + """Write the comparison report to stdout. + + Parameters + ---------- + report_data: + Populated report object from ``compare.build_report_data()``. + as_json: + When ``True``, serialise via ``ReportData.to_dict()`` and emit + compact JSON. When ``False``, emit the text rendering. + quiet: + When ``True`` and *as_json* is ``False``, suppress all stdout + output (useful for pipelines that only inspect the exit code). + """ + if as_json: + print(_json.dumps(report_data.to_dict(), indent=2, default=_json_default)) # noqa: T201 + elif not quiet: + print(report_data.render()) # noqa: T201 + + +def print_error(msg: str) -> None: + """Write *msg* to stderr with a ``datacompy:`` prefix.""" + print(f"datacompy: {msg}", file=sys.stderr) # noqa: T201 diff --git a/datacompy/cli/parser.py b/datacompy/cli/parser.py new file mode 100644 index 00000000..70ea45a9 --- /dev/null +++ b/datacompy/cli/parser.py @@ -0,0 +1,278 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Argument parser for the DataComPy CLI.""" + +import argparse +from pathlib import Path + +import datacompy +from datacompy.cli.compare import run_compare + + +def _single_char_delimiter(value: str) -> str: + r"""Argparse ``type=`` that accepts a single-character delimiter. + + Translates ``\t`` to a real tab so both the shell literal ``$'\t'`` and + the string ``\t`` work identically. Rejects anything that is not exactly + one character after translation. + """ + translated = value.replace("\\t", "\t") + if len(translated) != 1: + raise argparse.ArgumentTypeError( + f"--csv-delimiter must be a single character, got {value!r}" + ) + return translated + + +def _non_negative_int(value: str) -> int: + """Argparse ``type=`` that accepts a non-negative integer.""" + try: + n = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError( + f"--max-unequal-rows must be a non-negative integer, got {value!r}" + ) from exc + if n < 0: + raise argparse.ArgumentTypeError( + f"--max-unequal-rows must be a non-negative integer, got {n}" + ) + return n + + +def build_parser() -> argparse.ArgumentParser: + """Build and return the top-level argument parser.""" + parser = argparse.ArgumentParser( + prog="datacompy", + description="Compare two datasets across multiple backends.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {datacompy.__version__}", + ) + parser.add_argument( + "--debug", + action="store_true", + default=False, + help="Re-raise unexpected exceptions instead of printing a friendly message. " + "Useful when filing bug reports.", + ) + + sub = parser.add_subparsers(dest="command", required=True) + _add_compare_subparser(sub) + return parser + + +def _add_compare_subparser( + sub: "argparse._SubParsersAction[argparse.ArgumentParser]", +) -> None: + """Register the ``compare`` subcommand on *sub*.""" + cmp = sub.add_parser( + "compare", + help="Compare two datasets and report differences.", + description=( + "Load --left and --right files, compare them using --backend, " + "and exit 0 (match), 1 (mismatch / threshold violated), or 2 (error)." + ), + ) + + # ---- inputs ---------------------------------------------------------------- + inp = cmp.add_argument_group("input") + inp.add_argument( + "--left", required=True, metavar="PATH", help="Path or URI to the left dataset." + ) + inp.add_argument( + "--right", + required=True, + metavar="PATH", + help="Path or URI to the right dataset.", + ) + inp.add_argument( + "--format", + choices=["csv", "parquet", "json"], + default=None, + help="Input file format. Inferred from extension when omitted.", + ) + inp.add_argument( + "--csv-delimiter", + type=_single_char_delimiter, + default=",", + metavar="CHAR", + help=( + "Field delimiter for CSV files (default: comma). " + "Use '\\t' for tab-separated files (the shell escape $'\\t' also works)." + ), + ) + + # ---- join keys ------------------------------------------------------------- + keys = cmp.add_argument_group("join keys") + keys.add_argument( + "--on", + action="append", + dest="on", + default=None, + metavar="COL", + help="Join column name (required unless --on-index is used). Repeat for composite keys: --on id --on date.", + ) + keys.add_argument( + "--on-index", + action="store_true", + default=False, + help="Join on DataFrame index (Pandas backend only). Mutually exclusive with --on.", + ) + + # ---- backend --------------------------------------------------------------- + backend = cmp.add_argument_group("backend") + backend.add_argument( + "--backend", + choices=["pandas", "polars", "spark", "snowflake"], + default="polars", + help="Comparison backend. Polars is fast and the default; use Pandas for index-based joins or wider ecosystem compatibility. Default: polars.", + ) + + # ---- tolerances & flags ---------------------------------------------------- + tol = cmp.add_argument_group("tolerances and flags") + tol.add_argument( + "--abs-tol", + type=float, + default=0.0, + metavar="N", + help="Absolute tolerance for numeric comparisons (default 0.0).", + ) + tol.add_argument( + "--rel-tol", + type=float, + default=0.0, + metavar="N", + help="Relative tolerance for numeric comparisons (default 0.0).", + ) + tol.add_argument( + "--ignore-spaces", + action="store_true", + default=False, + help="Ignore leading/trailing whitespace in string columns.", + ) + tol.add_argument( + "--ignore-case", + action="store_true", + default=False, + help="Ignore case in string columns.", + ) + tol.add_argument( + "--ignore-extra-columns", + action="store_true", + default=False, + help="Treat comparisons as matching even if one side has extra columns.", + ) + tol.add_argument( + "--cast-column-names-lower", + action=argparse.BooleanOptionalAction, + default=True, + help="Cast column names to lowercase before comparing (default: enabled).", + ) + + # ---- naming ---------------------------------------------------------------- + names = cmp.add_argument_group("naming") + names.add_argument( + "--df1-name", + default=None, + metavar="NAME", + help="Label for the left dataset in the report (default: left filename stem).", + ) + names.add_argument( + "--df2-name", + default=None, + metavar="NAME", + help="Label for the right dataset in the report (default: right filename stem).", + ) + + # ---- report shape ---------------------------------------------------------- + report = cmp.add_argument_group("report") + report.add_argument( + "--sample-count", + type=int, + default=10, + metavar="N", + help="Max mismatch rows to sample per column (default 10).", + ) + report.add_argument( + "--column-count", + type=int, + default=10, + metavar="N", + help="Max columns to display in unique-row samples (default 10).", + ) + report.add_argument( + "--max-unequal-rows", + type=_non_negative_int, + default=None, + metavar="N", + help=( + "Exit 0 if the number of differing rows is at most N; exit 1 otherwise. " + "By default counts both value mismatches AND rows that exist only in one " + "dataset. Pass --ignore-unique-rows to count value mismatches only. " + "Must be a non-negative integer." + ), + ) + report.add_argument( + "--ignore-unique-rows", + action="store_true", + default=False, + help=( + "With --max-unequal-rows: exclude rows that exist only in one dataset " + "from the difference count. Only value mismatches in common rows are counted." + ), + ) + + # ---- output ---------------------------------------------------------------- + out = cmp.add_argument_group("output") + out.add_argument( + "--json", + action="store_true", + default=False, + help="Emit JSON report to stdout instead of text.", + ) + out.add_argument( + "--quiet", + action="store_true", + default=False, + help=( + "Suppress the text report. Has no effect when --json is also given. " + "Exit code still reflects match/mismatch." + ), + ) + + # ---- backend-specific ------------------------------------------------------ + bspec = cmp.add_argument_group("backend-specific options") + bspec.add_argument( + "--spark-app-name", + default="datacompy-cli", + metavar="NAME", + help="Spark application name (Spark backend only).", + ) + bspec.add_argument( + "--snowflake-config", + type=Path, + default=None, + metavar="PATH", + help="Path to a JSON file with Snowflake connection parameters. " + "Overrides environment variables: SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, " + "SNOWFLAKE_PASSWORD (or SNOWFLAKE_AUTHENTICATOR for SSO), and optionally " + "SNOWFLAKE_ROLE, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_DATABASE, SNOWFLAKE_SCHEMA " + "(Snowflake backend only).", + ) + + cmp.set_defaults(func=run_compare) diff --git a/datacompy/cli/sessions.py b/datacompy/cli/sessions.py new file mode 100644 index 00000000..400ec034 --- /dev/null +++ b/datacompy/cli/sessions.py @@ -0,0 +1,146 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Session factories for optional backends (Spark, Snowflake). + +Imports for ``pyspark`` and ``snowflake.snowpark`` are quarantined here +so they are never loaded at package import time — the same pattern used +in ``datacompy/__init__.py``. +""" + +import json +import os +from pathlib import Path +from typing import Any + +from datacompy.cli.errors import BadArgsError, MissingExtraError + + +def get_spark_session(app_name: str = "datacompy-cli") -> Any: + """Return a :class:`pyspark.sql.SparkSession`, creating one if needed. + + Sets log level to ``ERROR`` by default to suppress PySpark's verbose + INFO/WARN output from the CLI stdout stream. Override by setting the + ``DATACOMPY_SPARK_LOG_LEVEL`` environment variable (e.g. ``INFO``). + + Parameters + ---------- + app_name: + Spark application name passed to ``SparkSession.builder.appName``. + + Raises + ------ + MissingExtraError + When ``pyspark`` is not installed. + """ + try: + from pyspark.sql import SparkSession + except ImportError as exc: + raise MissingExtraError( + "Spark backend requires 'datacompy[spark]'. " + "Install it with: pip install datacompy[spark]" + ) from exc + + spark = SparkSession.builder.appName(app_name).getOrCreate() + log_level = os.environ.get("DATACOMPY_SPARK_LOG_LEVEL", "ERROR") + spark.sparkContext.setLogLevel(log_level) + return spark + + +def get_snowflake_session(config_path: Path | None = None) -> Any: + """Return a :class:`snowflake.snowpark.Session`. + + Parameters + ---------- + config_path: + Optional path to a JSON file whose top-level keys are Snowflake + connection parameters (``account``, ``user``, ``password``, etc.). + When ``None`` the session is built from environment variables: + + - ``SNOWFLAKE_ACCOUNT`` (required) + - ``SNOWFLAKE_USER`` (required) + - ``SNOWFLAKE_PASSWORD`` (required unless ``SNOWFLAKE_AUTHENTICATOR`` + is set to ``externalbrowser`` or another SSO authenticator) + - ``SNOWFLAKE_ROLE`` (optional) + - ``SNOWFLAKE_WAREHOUSE`` (optional) + - ``SNOWFLAKE_DATABASE`` (optional) + - ``SNOWFLAKE_SCHEMA`` (optional) + - ``SNOWFLAKE_AUTHENTICATOR`` (optional; e.g. ``externalbrowser``) + + Raises + ------ + MissingExtraError + When ``snowflake.snowpark`` is not installed. + BadArgsError + When required environment variables are missing. + """ + try: + from snowflake.snowpark.session import Session + except ImportError as exc: + raise MissingExtraError( + "Snowflake backend requires 'datacompy[snowflake]'. " + "Install it with: pip install datacompy[snowflake]" + ) from exc + + if config_path is not None: + try: + params: dict[str, str] = json.loads(config_path.read_text()) + except FileNotFoundError as exc: + raise BadArgsError( + f"--snowflake-config file not found: {config_path}" + ) from exc + except (OSError, json.JSONDecodeError) as exc: + raise BadArgsError(f"--snowflake-config {config_path}: {exc}") from exc + return Session.builder.configs(params).create() + + # Build from environment variables. + account = os.environ.get("SNOWFLAKE_ACCOUNT") + user = os.environ.get("SNOWFLAKE_USER") + password = os.environ.get("SNOWFLAKE_PASSWORD") + authenticator = os.environ.get("SNOWFLAKE_AUTHENTICATOR") + + missing = [ + name + for name, val in [("SNOWFLAKE_ACCOUNT", account), ("SNOWFLAKE_USER", user)] + if not val + ] + if missing: + raise BadArgsError( + f"Missing required environment variable(s): {', '.join(missing)}. " + "Set them or pass --snowflake-config path/to/conn.json." + ) + + if not password and not authenticator: + raise BadArgsError( + "Either SNOWFLAKE_PASSWORD or SNOWFLAKE_AUTHENTICATOR must be set. " + "Pass --snowflake-config for full connection parameter control." + ) + + params = {"account": account, "user": user} # type: ignore[dict-item] + if password: + params["password"] = password + if authenticator: + params["authenticator"] = authenticator + for env_var, key in [ + ("SNOWFLAKE_ROLE", "role"), + ("SNOWFLAKE_WAREHOUSE", "warehouse"), + ("SNOWFLAKE_DATABASE", "database"), + ("SNOWFLAKE_SCHEMA", "schema"), + ]: + val = os.environ.get(env_var) + if val: + params[key] = val + + return Session.builder.configs(params).create() diff --git a/docs/source/cli.rst b/docs/source/cli.rst new file mode 100644 index 00000000..9f0f488f --- /dev/null +++ b/docs/source/cli.rst @@ -0,0 +1,319 @@ +CLI Usage +========= + +DataComPy ships a ``datacompy`` command-line tool for comparing two datasets +without writing a Python script. It is installed automatically alongside the +library: + +.. code-block:: bash + + pip install datacompy + +Quickstart +---------- + +.. code-block:: bash + + # Compare two CSV files (Polars backend, default) + datacompy compare --left before.csv --right after.csv --on id + + # Exit codes: 0 = match, 1 = mismatch, 2 = error + echo $? + + # Python module form (useful when the script is not on PATH) + python -m datacompy.cli compare --left before.csv --right after.csv --on id + +Subcommands +----------- + +``compare`` +~~~~~~~~~~~ + +.. code-block:: text + + datacompy compare [OPTIONS] + +Input flags +^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--left PATH`` + - Path or URI to the left (reference) dataset. **Required.** + * - ``--right PATH`` + - Path or URI to the right (candidate) dataset. **Required.** + * - ``--format {csv,parquet,json}`` + - Override format detection. By default the format is inferred from + the file extension (``*.csv`` / ``*.tsv``, ``*.parquet`` / ``*.pq``, + ``*.json`` / ``*.jsonl``). + * - ``--csv-delimiter CHAR`` + - Field delimiter for CSV files (default: comma). Use ``';'`` for + European CSVs or ``'\\t'`` for TSV files. Both the escape sequence + ``'\\t'`` and a literal tab character are accepted. + +Join-key flags +^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--on COL`` + - Join column name. Repeat for composite keys: + ``--on id --on date``. + * - ``--on-index`` + - Join on the DataFrame index instead of columns. + **Pandas backend only.** Mutually exclusive with ``--on``. + +Backend +^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--backend {pandas,polars,spark,snowflake}`` + - Comparison backend (default: **polars**). Polars is faster and + ships as a hard dependency. Spark and Snowflake require their + respective optional extras (``datacompy[spark]``, + ``datacompy[snowflake]``). + +Tolerances and comparison flags +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--abs-tol N`` + - Absolute tolerance for numeric comparisons (default 0.0). + * - ``--rel-tol N`` + - Relative tolerance for numeric comparisons (default 0.0). + * - ``--ignore-spaces`` + - Strip leading / trailing whitespace from string columns. + * - ``--ignore-case`` + - Treat string comparisons as case-insensitive. + * - ``--ignore-extra-columns`` + - Pass even if one dataset has columns the other lacks. + * - ``--cast-column-names-lower`` / ``--no-cast-column-names-lower`` + - Normalise column names to lowercase before comparing + (enabled by default). + +Naming +^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--df1-name NAME`` + - Label for the left dataset in the report. Defaults to the filename + stem for file paths (``sales_data.csv`` → ``sales_data``) or the + table name segment for Snowflake refs + (``PROD.ANALYTICS.SALES_FACT`` → ``SALES_FACT``). + * - ``--df2-name NAME`` + - Label for the right dataset in the report. Same defaulting rules + as ``--df1-name``. + +Report shape +^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--sample-count N`` + - Maximum mismatch rows to sample per column (default 10). + * - ``--column-count N`` + - Maximum columns to display in unique-row samples (default 10). + * - ``--max-unequal-rows N`` + - Exit 0 if total differing rows ≤ *N*; exit 1 otherwise. By default + counts both value mismatches **and** rows that exist only in one + dataset. Must be a non-negative integer. + * - ``--ignore-unique-rows`` + - With ``--max-unequal-rows``: exclude rows that exist only in one + dataset from the count. Only value mismatches in common rows are + counted. + +Output +^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Flag + - Description + * - ``--json`` + - Emit the full comparison report as a JSON object to stdout. + * - ``--quiet`` + - Suppress all stdout output. The exit code still reflects the + comparison result. Has no effect when ``--json`` is also given + (JSON is always emitted when ``--json`` is set). + +To write a report to a file, use shell redirection: + +.. code-block:: bash + + datacompy compare --left a.csv --right b.csv --on id > report.txt + datacompy compare --left a.csv --right b.csv --on id --json > report.json + +Exit codes +---------- + +.. list-table:: + :widths: 10 90 + :header-rows: 1 + + * - Code + - Meaning + * - ``0`` + - The datasets match (within any specified tolerances / thresholds). + * - ``1`` + - The datasets differ, or ``--max-unequal-rows`` was violated. + * - ``2`` + - An error occurred (bad arguments, missing file, import error, etc.). + +Backend-specific notes +----------------------- + +Spark +~~~~~ + +The Spark backend builds a local ``SparkSession`` via +``SparkSession.builder.getOrCreate()``. Java 17 must be on ``PATH``. +PySpark's INFO/WARN output is suppressed at the ``ERROR`` level by default; +set ``DATACOMPY_SPARK_LOG_LEVEL=INFO`` to restore verbose logs. + +Install the Spark extra before using ``--backend spark``: + +.. code-block:: bash + + pip install "datacompy[spark]" + +Snowflake +~~~~~~~~~ + +The Snowflake backend builds a Snowpark session from environment variables: + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Variable + - Notes + * - ``SNOWFLAKE_ACCOUNT`` + - **Required.** + * - ``SNOWFLAKE_USER`` + - **Required.** + * - ``SNOWFLAKE_PASSWORD`` + - Required unless ``SNOWFLAKE_AUTHENTICATOR=externalbrowser``. + * - ``SNOWFLAKE_AUTHENTICATOR`` + - Optional. Set to ``externalbrowser`` for SSO. + * - ``SNOWFLAKE_ROLE`` + - Optional. + * - ``SNOWFLAKE_WAREHOUSE`` + - Optional. + * - ``SNOWFLAKE_DATABASE`` + - Optional. + * - ``SNOWFLAKE_SCHEMA`` + - Optional. + +Override all variables with a JSON connection-parameter file: + +.. code-block:: bash + + datacompy compare \ + --left DB.SCHEMA.TABLE_BEFORE \ + --right DB.SCHEMA.TABLE_AFTER \ + --on ID \ + --backend snowflake \ + --snowflake-config ~/.snowflake/conn.json + +Pass ``DB.SCHEMA.TABLE`` (fully-qualified) or ``SCHEMA.TABLE`` (2-part) +identifiers directly as ``--left`` / ``--right`` values. When a 2-part +ref is given, the CLI expands it to 3 parts using the session's current +database (set via ``SNOWFLAKE_DATABASE`` or the connection config file). +If no current database is set and a 2-part ref is used, the CLI exits +with an error asking you to use the fully-qualified form. +Local files are uploaded to temporary Snowflake tables automatically. + +Install the Snowflake extra: + +.. code-block:: bash + + pip install "datacompy[snowflake]" + +Cloud paths +~~~~~~~~~~~ + +``s3://``, ``gs://``, and ``abfs://`` URIs are forwarded as-is to the +Pandas / Polars reader. They require optional filesystem libraries: + +.. code-block:: bash + + pip install s3fs # Amazon S3 + pip install gcsfs # Google Cloud Storage + pip install adlfs # Azure Data Lake Storage + +CICD recipes +------------ + +GitHub Actions +~~~~~~~~~~~~~~ + +.. code-block:: yaml + + - name: Compare datasets + run: | + datacompy compare \ + --left data/expected.csv \ + --right data/actual.csv \ + --on id \ + --max-unequal-rows 0 + +Airflow BashOperator +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + compare_task = BashOperator( + task_id="compare_datasets", + bash_command=( + "datacompy compare " + "--left {{ params.left }} " + "--right {{ params.right }} " + "--on id --json > /tmp/report.json" + ), + params={"left": "s3://bucket/before.parquet", "right": "s3://bucket/after.parquet"}, + ) + +Limitations +----------- + +- **Per-column tolerances** are not yet supported via the CLI. Use the + Python API to pass ``abs_tol={"col_a": 0.01}`` directly. +- **File output** (e.g. ``--output report.html``) is not yet supported. + Use shell redirection (``> report.txt``, ``> report.json``) instead. +- **Mixed file formats** — ``--format`` applies to both sides. If your + left and right files have different formats (e.g. a CSV reference vs a + Parquet snapshot), convert one before comparing. Per-side + ``--left-format`` / ``--right-format`` flags are planned for a future + release. +- **Empty DataFrames** — comparing two datasets that both have zero rows + currently exits 1 (mismatch) due to a known behaviour in the underlying + comparison engine. This will be addressed in a future library release. diff --git a/docs/source/index.rst b/docs/source/index.rst index 46ee380a..8dbc541b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,6 +16,7 @@ Contents Polars Usage Template Guide Report API + CLI Usage Benchmarks Developer Instructions diff --git a/pyproject.toml b/pyproject.toml index a3a5272c..8ad54afb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,15 +28,14 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", ] dynamic = [ "version" ] dependencies = [ "jinja2>=3", - "numpy<2.5,>=1.26.4", - "ordered-set<=4.1,>=4.0.2", - "pandas<3.1,>=2.2", - "polars[pandas]<1.42,>=0.20.4", + "numpy>=1.26.4,<2.5", + "ordered-set>=4.0.2,<=4.1", + "pandas>=2.2,<3.1", + "polars[pandas]>=0.20.4,<1.42", ] optional-dependencies.build = [ "build", "twine", "wheel" ] optional-dependencies.dev = [ @@ -51,26 +50,33 @@ optional-dependencies.dev = [ optional-dependencies.docs = [ "furo", "myst-parser", "sphinx" ] optional-dependencies.edgetest = [ "edgetest", "edgetest-conda" ] optional-dependencies.qa = [ "mypy", "pandas-stubs", "pre-commit", "ruff" ] -optional-dependencies.snowflake = [ "snowflake-snowpark-python<=1.50.1,>=1.37" ] +optional-dependencies.snowflake = [ "snowflake-snowpark-python>=1.37,<=1.50.1" ] optional-dependencies.spark = [ - "pyspark[connect]!=4,<=4.1.1,>=3.5; python_version<='3.11'", + "pyspark[connect]>=3.5,!=4,<=4.1.1; python_version<='3.11'", "pyspark[connect]>=4; python_version>='3.12'", ] optional-dependencies.tests = [ "pytest", "pytest-benchmark", "pytest-cov", - "pytest-profiling" + "pytest-profiling", ] optional-dependencies.tests-spark = [ "pytest-spark" ] urls."Bug Tracker" = "https://github.com/capitalone/datacompy/issues" -urls."Source Code" = "https://github.com/capitalone/datacompy" urls.Documentation = "https://capitalone.github.io/datacompy/" urls.Homepage = "https://github.com/capitalone/datacompy" urls.Repository = "https://github.com/capitalone/datacompy.git" +urls."Source Code" = "https://github.com/capitalone/datacompy" +scripts.datacompy = "datacompy.cli:main" + [tool.setuptools] -packages = [ "datacompy", "datacompy.comparator", "datacompy.templates" ] +packages = [ + "datacompy", + "datacompy.cli", + "datacompy.comparator", + "datacompy.templates", +] zip-safe = false include-package-data = true dynamic.version = { attr = "datacompy.__version__" } @@ -84,18 +90,18 @@ target-version = "py312" src = [ "src" ] extend-include = [ "*.ipynb" ] lint.select = [ - "B", # flake8-bugbear + "B", # flake8-bugbear # "A", # flake8-builtins - "C4", # flake8-comprehensions - "D", # pydocstyle - "E", # pycodestyle errors - "F", # pyflakes - "I", # isort - "LOG", # flake8-logging - "NPY", # numpy rules - "RUF", # Ruff errors + "C4", # flake8-comprehensions + "D", # pydocstyle + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "LOG", # flake8-logging + "NPY", # numpy rules + "RUF", # Ruff errors # "ARG", # flake8-unused-arguments - "SIM", # flake8-simplify + "SIM", # flake8-simplify # "C901", # mccabe complexity # "G", # flake8-logging-format "T20", # flake8-print @@ -130,6 +136,8 @@ column_width = 88 strict = true overrides = [ { module = "pyarrow", ignore_missing_imports = true }, + { module = "pyspark.*", ignore_missing_imports = true }, + { module = "snowflake.*", ignore_missing_imports = true }, ] [edgetest.envs.core] diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 00000000..bf36ff12 --- /dev/null +++ b/tests/cli/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py new file mode 100644 index 00000000..93db8067 --- /dev/null +++ b/tests/cli/conftest.py @@ -0,0 +1,136 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared fixtures for CLI tests.""" + +import csv +from collections.abc import Callable, Generator +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from datacompy.cli import main + + +@pytest.fixture() +def cli( + capsys: pytest.CaptureFixture[str], +) -> Callable[[list[str]], tuple[int, str, str]]: + """Return a callable that runs ``main(argv)`` and returns ``(exit_code, stdout, stderr)``.""" + + def _run(argv: list[str]) -> tuple[int, str, str]: + try: + code = main(argv) + except SystemExit as exc: + code = int(exc.code) if isinstance(exc.code, int) else 2 + captured = capsys.readouterr() + return code, captured.out, captured.err + + return _run + + +@pytest.fixture() +def csv_match(tmp_path: Path) -> Generator[tuple[Path, Path], None, None]: + """Yield two identical 2-row CSVs ``(left, right)``.""" + rows = [["id", "val"], ["1", "a"], ["2", "b"]] + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + for p in (left, right): + with p.open("w", newline="") as f: + csv.writer(f).writerows(rows) + yield left, right + + +@pytest.fixture() +def csv_mismatch(tmp_path: Path) -> Generator[tuple[Path, Path], None, None]: + """Yield two 2-row CSVs that differ on row 2 ``(left, right)``.""" + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + with left.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "b"]]) + with right.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "X"]]) + yield left, right + + +@pytest.fixture() +def csv_pair( + tmp_path: Path, +) -> Callable[[str, str], tuple[Path, Path]]: + """Return a factory that writes two CSV strings to ``left.csv`` / ``right.csv``.""" + + def _make(left_text: str, right_text: str) -> tuple[Path, Path]: + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + left.write_text(left_text) + right.write_text(right_text) + return left, right + + return _make + + +@pytest.fixture() +def tsv_pair( + tmp_path: Path, +) -> Callable[[list[list[str]], list[list[str]]], tuple[Path, Path]]: + """Return a factory that writes two TSV files to ``left.csv`` / ``right.csv``.""" + + def _make( + left_rows: list[list[str]], right_rows: list[list[str]] + ) -> tuple[Path, Path]: + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + for path, rows in ((left, left_rows), (right, right_rows)): + with path.open("w", newline="") as f: + csv.writer(f, delimiter="\t").writerows(rows) + return left, right + + return _make + + +@pytest.fixture() +def mock_snowflake_session() -> MagicMock: + """Return a MagicMock Snowpark session with db=PROD, schema=PUBLIC.""" + sess = MagicMock() + sess.get_current_database.return_value = "PROD" + sess.get_current_schema.return_value = "PUBLIC" + sess.write_pandas.return_value = None + return sess + + +@pytest.fixture() +def mock_snowflake_compare() -> Callable[[bool], MagicMock]: + """Return a factory that produces a pre-configured ``MagicMock`` compare object. + + Usage:: + + def test_something(mock_snowflake_compare, cli): + mock_compare = mock_snowflake_compare(matches=True) + with patch("datacompy.cli.compare.make_snowflake_compare", return_value=mock_compare): + ... + """ + + def _factory(matches: bool = True) -> MagicMock: + mock_compare = MagicMock() + mock_compare.build_report_data.return_value = MagicMock( + render=MagicMock( + return_value="DataComPy Comparison\nMatch: " + str(matches) + ), + row_summary=MagicMock(unequal_rows=0 if matches else 1), + ) + mock_compare.matches.return_value = matches + return mock_compare + + return _factory diff --git a/tests/cli/test_compare_pandas.py b/tests/cli/test_compare_pandas.py new file mode 100644 index 00000000..64b2f80b --- /dev/null +++ b/tests/cli/test_compare_pandas.py @@ -0,0 +1,516 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI integration tests for the Pandas backend.""" + +import csv +import json +from collections.abc import Callable +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + + +def test_match_exits_0( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 0 + assert "DataComPy Comparison" in out + + +def test_mismatch_exits_1( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 1 + + +def test_json_output_is_valid( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--json", + ] + ) + assert code == 1 + data = json.loads(out) + assert "row_summary" in data + assert data["row_summary"]["unequal_rows"] == 1 + + +def test_quiet_suppresses_output( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--quiet", + ] + ) + assert code == 0 + assert out == "" + + +def test_max_unequal_rows_threshold_fail( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--max-unequal-rows", + "0", + ] + ) + assert code == 1 + + +def test_max_unequal_rows_threshold_pass( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--max-unequal-rows", + "9999", + ] + ) + assert code == 0 + + +def test_missing_left_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + right = tmp_path / "right.csv" + right.write_text("id,val\n1,a\n") + code, _, err = cli( + [ + "compare", + "--left", + str(tmp_path / "missing.csv"), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 2 + assert "missing.csv" in err or "not found" in err.lower() + + +def test_on_index_with_polars_exits_2( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on-index", + "--backend", + "polars", + ] + ) + assert code == 2 + assert "--on-index" in err + + +def test_on_index_and_on_mutually_exclusive_exits_2( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--on-index", + "--backend", + "pandas", + ] + ) + assert code == 2 + + +def test_on_index_pandas_match( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + content = pd.DataFrame({"val": ["a", "b"]}).to_csv(index=False) + left, right = csv_pair(content, content) + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on-index", + "--backend", + "pandas", + ] + ) + assert code == 0 + + +def test_parquet_input( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left = tmp_path / "left.parquet" + right = tmp_path / "right.parquet" + table = pa.table({"id": [1, 2], "val": ["a", "b"]}) + pq.write_table(table, left) + pq.write_table(table, right) + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 0 + + +def test_unknown_extension_without_format_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n", "id,val\n1,a\n") + left = left.rename(left.parent / "data.xyz") + right = right.rename(right.parent / "data2.xyz") + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 2 + assert ".xyz" in err + + +def test_pandas_no_join_key_exits_2( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, _, err = cli( + ["compare", "--left", str(left), "--right", str(right), "--backend", "pandas"] + ) + assert code == 2 + assert "--on" in err + + +def test_negative_max_unequal_rows_exits_2( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--max-unequal-rows", + "-1", + ] + ) + assert code == 2 + assert "non-negative" in err + + +def test_csv_delimiter_semicolon( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id;val\n1;a\n2;b\n", "id;val\n1;a\n2;b\n") + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + ";", + ] + ) + assert code == 0 + + +def test_txt_extension_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n", "id,val\n1,a\n") + left = left.rename(left.parent / "data.txt") + right = right.rename(right.parent / "data2.txt") + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert code == 2 + assert ".txt" in err + + +def test_df_names_default_to_stem( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left = tmp_path / "sales_before.csv" + right = tmp_path / "sales_after.csv" + for p in (left, right): + with p.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "b"]]) + _, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + ] + ) + assert "sales_before" in out + assert "sales_after" in out + + +# --------------------------------------------------------------------------- +# --csv-delimiter validation +# --------------------------------------------------------------------------- + + +def test_multi_char_delimiter_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n", "id,val\n1,a\n") + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "||", + ] + ) + assert code == 2 + assert "csv-delimiter" in err.lower() or "delimiter" in err.lower() + + +def test_empty_delimiter_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n", "id,val\n1,a\n") + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "", + ] + ) + assert code == 2 + assert "csv-delimiter" in err.lower() or "delimiter" in err.lower() + + +def test_single_char_delimiter_is_accepted( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id|val\n1|a\n2|b\n", "id|val\n1|a\n2|b\n") + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "|", + ] + ) + assert code == 0 + + +# --------------------------------------------------------------------------- +# --debug flag +# --------------------------------------------------------------------------- + + +def test_debug_flag_re_raises_cli_error( + tmp_path: Path, +) -> None: + """--debug must cause CLIError subclasses to propagate as real exceptions + rather than being swallowed into a friendly exit-2 message.""" + from datacompy.cli import main + from datacompy.cli.errors import BadArgsError + + with pytest.raises(BadArgsError): + main( + [ + "--debug", + "compare", + "--left", + str(tmp_path / "a.csv"), + "--right", + str(tmp_path / "b.csv"), + "--on", + "id", + "--on-index", # mutually exclusive with --on -- triggers BadArgsError + "--backend", + "pandas", + ] + ) diff --git a/tests/cli/test_compare_polars.py b/tests/cli/test_compare_polars.py new file mode 100644 index 00000000..1ab10400 --- /dev/null +++ b/tests/cli/test_compare_polars.py @@ -0,0 +1,322 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI integration tests for the Polars backend (default).""" + +import csv +import json +from collections.abc import Callable +from pathlib import Path + +import pytest + + +def test_match_exits_0_default_backend( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """Polars is the default backend -- no --backend flag needed.""" + left, right = csv_match + code, out, _ = cli( + ["compare", "--left", str(left), "--right", str(right), "--on", "id"] + ) + assert code == 0 + assert "DataComPy Comparison" in out + + +def test_mismatch_exits_1( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, _, _ = cli( + ["compare", "--left", str(left), "--right", str(right), "--on", "id"] + ) + assert code == 1 + + +def test_json_output( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, out, _ = cli( + ["compare", "--left", str(left), "--right", str(right), "--on", "id", "--json"] + ) + assert code == 1 + data = json.loads(out) + assert data["row_summary"]["unequal_rows"] == 1 + + +def test_missing_on_exits_2( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, _, err = cli(["compare", "--left", str(left), "--right", str(right)]) + assert code == 2 + assert "--on" in err + + +def test_df_names_appear_in_output( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + _, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--df1-name", + "before", + "--df2-name", + "after", + ] + ) + assert "before" in out + assert "after" in out + + +def test_missing_left_exits_2( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + right = tmp_path / "right.csv" + right.write_text("id,val\n1,a\n") + code, _, err = cli( + [ + "compare", + "--left", + str(tmp_path / "missing.csv"), + "--right", + str(right), + "--on", + "id", + ] + ) + assert code == 2 + assert "missing.csv" in err or "not found" in err.lower() + + +def test_max_unequal_rows_counts_unique_rows( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + with left.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "b"], ["3", "c"]]) + right.write_text("id,val\n1,a\n") + # unequal_rows=0 but df1_unique=2, so total_diff=2 > 0 → exit 1 + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--max-unequal-rows", + "0", + ] + ) + assert code == 1 + + +def test_ignore_unique_rows_flag( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + with left.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "b"], ["3", "c"]]) + right.write_text("id,val\n1,a\n") + # unequal_rows=0, unique rows excluded → total_diff=0 ≤ 0 → exit 0 + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--max-unequal-rows", + "0", + "--ignore-unique-rows", + ] + ) + assert code == 0 + + +def test_max_unequal_rows_with_ignore_extra_columns( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n2,b\n", "id,val,extra\n1,a,x\n2,b,y\n") + # extra column in right, values match, --ignore-extra-columns → exit 0 + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--max-unequal-rows", + "0", + "--ignore-extra-columns", + ] + ) + assert code == 0 + + +def test_df_names_default_to_stem( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left = tmp_path / "before_snapshot.csv" + right = tmp_path / "after_snapshot.csv" + for p in (left, right): + with p.open("w", newline="") as f: + csv.writer(f).writerows([["id", "val"], ["1", "a"], ["2", "b"]]) + _, out, _ = cli( + ["compare", "--left", str(left), "--right", str(right), "--on", "id"] + ) + assert "before_snapshot" in out + assert "after_snapshot" in out + + +def test_ignore_unique_rows_without_max_unequal_rows_exits_2( + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair("id,val\n1,a\n", "id,val\n1,a\n") + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--ignore-unique-rows", + ] + ) + assert code == 2 + assert "--max-unequal-rows" in err + + +# --------------------------------------------------------------------------- +# Tolerance and string-normalisation flags +# +# Each case: without the flag the comparison fails (exit 1); with it, passes. +# A separate "still fails" case verifies the flag does not mask real diffs. +# --------------------------------------------------------------------------- + +_FLAG_PASS_CASES = [ + pytest.param( + "id,val\n1,1.0\n2,2.0\n", + "id,val\n1,1.0\n2,2.005\n", + ["--abs-tol", "0.01"], + id="abs_tol_within_tolerance", + ), + pytest.param( + "id,val\n1,100.0\n", + "id,val\n1,101.0\n", + ["--rel-tol", "0.02"], + id="rel_tol_within_tolerance", + ), + pytest.param( + "id,val\n1,hello\n2,world\n", + "id,val\n1, hello \n2, world \n", + ["--ignore-spaces"], + id="ignore_spaces_whitespace_only_diff", + ), + pytest.param( + "id,val\n1,Hello\n2,World\n", + "id,val\n1,HELLO\n2,WORLD\n", + ["--ignore-case"], + id="ignore_case_case_only_diff", + ), +] + + +@pytest.mark.parametrize("left_csv,right_csv,flag", _FLAG_PASS_CASES) +def test_flag_turns_failing_comparison_into_pass( + left_csv: str, + right_csv: str, + flag: list[str], + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair(left_csv, right_csv) + base_argv = ["compare", "--left", str(left), "--right", str(right), "--on", "id"] + code_no_flag, _, _ = cli(base_argv) + assert code_no_flag == 1 + code_flag, _, _ = cli([*base_argv, *flag]) + assert code_flag == 0 + + +_FLAG_STILL_FAIL_CASES = [ + pytest.param( + "id,val\n1,1.0\n2,2.0\n", + "id,val\n1,1.0\n2,2.5\n", + ["--abs-tol", "0.01"], + id="abs_tol_exceeds_tolerance", + ), + pytest.param( + "id,val\n1,100.0\n", + "id,val\n1,120.0\n", + ["--rel-tol", "0.02"], + id="rel_tol_exceeds_tolerance", + ), + pytest.param( + "id,val\n1,hello\n", + "id,val\n1,world\n", + ["--ignore-spaces"], + id="ignore_spaces_non_whitespace_diff", + ), + pytest.param( + "id,val\n1,hello\n", + "id,val\n1,world\n", + ["--ignore-case"], + id="ignore_case_non_case_diff", + ), +] + + +@pytest.mark.parametrize("left_csv,right_csv,flag", _FLAG_STILL_FAIL_CASES) +def test_flag_does_not_mask_real_difference( + left_csv: str, + right_csv: str, + flag: list[str], + csv_pair: Callable[[str, str], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_pair(left_csv, right_csv) + code, _, _ = cli( + ["compare", "--left", str(left), "--right", str(right), "--on", "id", *flag] + ) + assert code == 1 diff --git a/tests/cli/test_compare_snowflake.py b/tests/cli/test_compare_snowflake.py new file mode 100644 index 00000000..bf3adaf8 --- /dev/null +++ b/tests/cli/test_compare_snowflake.py @@ -0,0 +1,197 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI integration tests for the Snowflake backend. + +Skipped automatically when snowflake.snowpark is not installed. + +The ``test_snowflake_missing_account_exits_2`` test runs without a real +Snowflake account (it only tests the error path in ``get_snowflake_session``). + +The comparison test ``test_snowflake_compare_match`` fully mocks the compare +layer so it works without any live Snowflake session. +""" + +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +snowpark = pytest.importorskip("snowflake.snowpark") + +from datacompy.cli.errors import BadArgsError +from datacompy.cli.loaders import _expand_table_ref +from datacompy.cli.sessions import get_snowflake_session + + +def test_snowflake_missing_account_exits_2( + monkeypatch: pytest.MonkeyPatch, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + for var in ( + "SNOWFLAKE_ACCOUNT", + "SNOWFLAKE_USER", + "SNOWFLAKE_PASSWORD", + "SNOWFLAKE_AUTHENTICATOR", + ): + monkeypatch.delenv(var, raising=False) + code, _, err = cli( + [ + "compare", + "--left", + "dummy_left.csv", + "--right", + "dummy_right.csv", + "--on", + "id", + "--backend", + "snowflake", + ] + ) + assert code == 2 + assert ( + "SNOWFLAKE_ACCOUNT" in err + or "SNOWFLAKE_USER" in err + or "missing" in err.lower() + ) + + +def test_snowflake_compare_match( + tmp_path: Path, + mock_snowflake_compare: Callable[[bool], MagicMock], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """Mock out the full compare stack so no live Snowflake session is needed.""" + mock_compare = mock_snowflake_compare(matches=True) + + left = tmp_path / "left.csv" + right = tmp_path / "right.csv" + left.write_text("id,val\n1,a\n") + right.write_text("id,val\n1,a\n") + + with ( + patch( + "datacompy.cli.sessions.get_snowflake_session", + return_value=MagicMock(), + ), + patch( + "datacompy.cli.compare.load_snowflake", + return_value="DB.SCHEMA.TABLE", + ), + patch( + "datacompy.cli.compare.make_snowflake_compare", + return_value=mock_compare, + ), + ): + code, out, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "snowflake", + ] + ) + + assert code == 0, f"exit code {code}, stderr: {err!r}" + assert "DataComPy Comparison" in out + + +def test_snowflake_two_part_ref_expanded() -> None: + """2-part schema.table ref is expanded to db.schema.table using session context.""" + mock_session = MagicMock() + mock_session.get_current_database.return_value = "PROD" + + assert ( + _expand_table_ref(mock_session, "ANALYTICS.SALES_FACT") + == "PROD.ANALYTICS.SALES_FACT" + ) + assert ( + _expand_table_ref(mock_session, "PROD.ANALYTICS.SALES_FACT") + == "PROD.ANALYTICS.SALES_FACT" + ) + + +def test_snowflake_two_part_ref_no_db_exits_2( + monkeypatch: pytest.MonkeyPatch, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """2-part ref with no current database on the session exits 2 with a clear message.""" + mock_session = MagicMock() + mock_session.get_current_database.return_value = None + + with patch( + "datacompy.cli.sessions.get_snowflake_session", return_value=mock_session + ): + code, _, err = cli( + [ + "compare", + "--left", + "ANALYTICS.SALES_FACT", + "--right", + "ANALYTICS.SALES_PREV", + "--on", + "ID", + "--backend", + "snowflake", + ] + ) + + assert code == 2 + assert "SNOWFLAKE_DATABASE" in err or "db.schema.table" in err + + +def test_missing_snowflake_config_raises_bad_args_error( + tmp_path: Path, +) -> None: + """get_snowflake_session must raise BadArgsError (not FileNotFoundError) + when --snowflake-config points to a non-existent file, so the caller + gets a message that names both the bad path and the flag to fix.""" + missing = tmp_path / "no_such_conn.json" + with pytest.raises(BadArgsError, match=r"no_such_conn\.json"): + get_snowflake_session(config_path=missing) + + +def test_missing_snowflake_config_file_exits_2_with_clear_message( + tmp_path: Path, + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """End-to-end: --snowflake-config pointing to a missing file must exit 2 + with a message that identifies the bad path.""" + missing = tmp_path / "no_such_conn.json" + + code, _, err = cli( + [ + "compare", + "--left", + "DB.SCHEMA.TABLE_L", + "--right", + "DB.SCHEMA.TABLE_R", + "--on", + "ID", + "--backend", + "snowflake", + "--snowflake-config", + str(missing), + ] + ) + + assert code == 2 + assert "no_such_conn.json" in err diff --git a/tests/cli/test_compare_spark.py b/tests/cli/test_compare_spark.py new file mode 100644 index 00000000..6d30abc3 --- /dev/null +++ b/tests/cli/test_compare_spark.py @@ -0,0 +1,69 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI integration tests for the Spark backend. + +Skipped automatically when PySpark is not installed. +""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +pyspark = pytest.importorskip("pyspark") + + +def test_spark_match_exits_0( + csv_match: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_match + code, out, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "spark", + ] + ) + assert code == 0 + assert "DataComPy Comparison" in out + + +def test_spark_mismatch_exits_1( + csv_mismatch: tuple[Path, Path], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left, right = csv_mismatch + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "spark", + ] + ) + assert code == 1 diff --git a/tests/cli/test_loaders.py b/tests/cli/test_loaders.py new file mode 100644 index 00000000..80834dab --- /dev/null +++ b/tests/cli/test_loaders.py @@ -0,0 +1,462 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for datacompy/cli/loaders.py and backends._default_name / _unescape_delimiter.""" + +import argparse +import importlib.util +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest +from datacompy.cli.backends import _default_name, to_compare_args +from datacompy.cli.errors import BadArgsError, LoadError +from datacompy.cli.loaders import ( + is_snowflake_ref, + load_pandas, + load_polars, + load_snowflake, +) +from datacompy.cli.parser import _single_char_delimiter + +# --------------------------------------------------------------------------- +# is_snowflake_ref +# --------------------------------------------------------------------------- + +_VALID_SNOWFLAKE_REFS = [ + pytest.param("PROD.ANALYTICS.SALES_FACT", id="three_part_upper"), + pytest.param("mydb.reporting.orders", id="three_part_lowercase"), + pytest.param("ANALYTICS.SALES_FACT", id="two_part"), + pytest.param("MY$DB.MY$SCHEMA.MY$TABLE", id="dollar_sign_in_segment"), + pytest.param("_db._schema._table", id="underscore_led_segment"), + pytest.param("PROD_2024.Analytics.SALES_FACT_V2", id="mixed_case_with_numbers"), +] + + +@pytest.mark.parametrize("ref", _VALID_SNOWFLAKE_REFS) +def test_is_snowflake_ref_valid(ref: str) -> None: + assert is_snowflake_ref(ref) is True + + +_INVALID_SNOWFLAKE_REFS = [ + pytest.param("sales_data.csv", id="csv_file"), + pytest.param("s3://bucket/prefix/file.csv", id="s3_uri"), + pytest.param("path/to/file.csv", id="slash_path"), + pytest.param('"My DB"."My Schema"."My Table"', id="quoted_identifier"), + pytest.param("mytable", id="single_segment"), + pytest.param("a.b.c.d", id="four_part"), +] + + +@pytest.mark.parametrize("ref", _INVALID_SNOWFLAKE_REFS) +def test_is_snowflake_ref_invalid(ref: str) -> None: + assert is_snowflake_ref(ref) is False + + +# --------------------------------------------------------------------------- +# _default_name +# --------------------------------------------------------------------------- + +_DEFAULT_NAME_CASES = [ + pytest.param("sales_data.csv", "sales_data", id="csv_file"), + pytest.param("archive/orders_2024.parquet", "orders_2024", id="parquet_file"), + pytest.param("s3://bucket/prefix/snapshot.csv", "snapshot", id="s3_uri"), + pytest.param("PROD.ANALYTICS.SALES_FACT", "SALES_FACT", id="three_part_ref"), + pytest.param( + "mydb.reporting.orders_2024", "orders_2024", id="three_part_lowercase_ref" + ), + pytest.param("ANALYTICS.SALES_FACT", "SALES_FACT", id="two_part_ref"), +] + + +@pytest.mark.parametrize("ref,expected", _DEFAULT_NAME_CASES) +def test_default_name(ref: str, expected: str) -> None: + assert _default_name(ref) == expected + + +def test_default_name_same_schema_different_tables_produce_distinct_names() -> None: + """Regression: Path.stem produced 'PROD.ANALYTICS' for both sides.""" + assert _default_name("PROD.ANALYTICS.SALES_FACT") != _default_name( + "PROD.ANALYTICS.SALES_PREV" + ) + assert _default_name("PROD.ANALYTICS.SALES_FACT") == "SALES_FACT" + assert _default_name("PROD.ANALYTICS.SALES_PREV") == "SALES_PREV" + + +def test_default_name_explicit_name_overrides_default(tmp_path: Path) -> None: + """to_compare_args respects --df1-name / --df2-name when provided.""" + ns = argparse.Namespace( + left="PROD.ANALYTICS.SALES_FACT", + right="PROD.ANALYTICS.SALES_PREV", + df1_name="before", + df2_name="after", + format=None, + on=["ID"], + on_index=False, + backend="snowflake", + abs_tol=0.0, + rel_tol=0.0, + ignore_spaces=False, + ignore_case=False, + ignore_extra_columns=False, + ignore_unique_rows=False, + cast_column_names_lower=True, + csv_delimiter=",", + sample_count=10, + column_count=10, + max_unequal_rows=None, + json=False, + quiet=False, + spark_app_name="datacompy-cli", + snowflake_config=None, + ) + args = to_compare_args(ns) + assert args.df1_name == "before" + assert args.df2_name == "after" + + +# --------------------------------------------------------------------------- +# _default_name integration: Snowflake report uses correct table names +# --------------------------------------------------------------------------- + + +def test_snowflake_three_part_refs_produce_distinct_df_names_in_report( + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """Verify the label reaches the report, not just _default_name in isolation.""" + mock_report = MagicMock() + mock_report.render.return_value = "DataComPy Comparison\nSALES_FACT vs SALES_PREV" + mock_report.row_summary = MagicMock(unequal_rows=0) + mock_compare = MagicMock() + mock_compare.build_report_data.return_value = mock_report + mock_compare.matches.return_value = True + + with ( + patch( + "datacompy.cli.sessions.get_snowflake_session", + return_value=MagicMock(), + ), + patch( + "datacompy.cli.compare.load_snowflake", + side_effect=lambda s, ref, fmt, **kw: ref, + ), + patch( + "datacompy.cli.compare.make_snowflake_compare", + return_value=mock_compare, + ) as mock_make, + ): + cli( + [ + "compare", + "--left", + "PROD.ANALYTICS.SALES_FACT", + "--right", + "PROD.ANALYTICS.SALES_PREV", + "--on", + "ID", + "--backend", + "snowflake", + ] + ) + + call_args = mock_make.call_args + built_args = call_args[0][0] + assert built_args.df1_name == "SALES_FACT" + assert built_args.df2_name == "SALES_PREV" + + +# --------------------------------------------------------------------------- +# load_pandas error handling +# --------------------------------------------------------------------------- + + +def test_load_pandas_missing_file_raises_load_error(tmp_path: Path) -> None: + with pytest.raises(LoadError, match="not found"): + load_pandas(str(tmp_path / "nonexistent.csv"), "csv") + + +def test_load_pandas_corrupt_parquet_raises_load_error(tmp_path: Path) -> None: + bad = tmp_path / "bad.parquet" + bad.write_bytes(b"this is not a valid parquet file") + with pytest.raises(LoadError, match=r"bad\.parquet"): + load_pandas(str(bad), "parquet") + + +def test_load_pandas_corrupt_json_raises_load_error(tmp_path: Path) -> None: + bad = tmp_path / "bad.json" + bad.write_text("{not valid json{{{{") + with pytest.raises(LoadError, match=r"bad\.json"): + load_pandas(str(bad), "json") + + +def test_load_pandas_binary_csv_raises_load_error(tmp_path: Path) -> None: + binary = tmp_path / "binary.csv" + binary.write_bytes(bytes(range(256))) + with pytest.raises(LoadError, match=r"binary\.csv"): + load_pandas(str(binary), "csv") + + +def test_load_pandas_error_message_contains_path(tmp_path: Path) -> None: + bad = tmp_path / "corrupt.parquet" + bad.write_bytes(b"garbage") + with pytest.raises(LoadError) as exc_info: + load_pandas(str(bad), "parquet") + assert "corrupt.parquet" in str(exc_info.value) + + +def test_load_pandas_valid_csv_loads_successfully(tmp_path: Path) -> None: + p = tmp_path / "good.csv" + p.write_text("id,val\n1,a\n2,b\n") + df = load_pandas(str(p), "csv") + assert list(df.columns) == ["id", "val"] + assert len(df) == 2 + + +# --------------------------------------------------------------------------- +# load_polars error handling +# --------------------------------------------------------------------------- + + +def test_load_polars_corrupt_parquet_raises_load_error(tmp_path: Path) -> None: + bad = tmp_path / "bad.parquet" + bad.write_bytes(b"garbage") + with pytest.raises(LoadError, match=r"bad\.parquet"): + load_polars(str(bad), "parquet") + + +def test_load_polars_missing_file_raises_load_error(tmp_path: Path) -> None: + with pytest.raises(LoadError, match="not found"): + load_polars(str(tmp_path / "nope.csv"), "csv") + + +# --------------------------------------------------------------------------- +# _single_char_delimiter (parser-level argparse type= callable) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw,expected", + [ + pytest.param("\\t", "\t", id="backslash_t_escape"), + pytest.param("\t", "\t", id="real_tab_unchanged"), + pytest.param(";", ";", id="semicolon_unchanged"), + pytest.param(",", ",", id="comma_unchanged"), + pytest.param("|", "|", id="pipe_unchanged"), + ], +) +def test_single_char_delimiter_valid(raw: str, expected: str) -> None: + assert _single_char_delimiter(raw) == expected + + +@pytest.mark.parametrize( + "raw", + [ + pytest.param("ab", id="two_chars"), + pytest.param("abc", id="three_chars"), + pytest.param("\\n", id="backslash_n_not_a_single_char"), + pytest.param("\\r", id="backslash_r_not_a_single_char"), + ], +) +def test_single_char_delimiter_invalid_raises(raw: str) -> None: + import argparse + + with pytest.raises(argparse.ArgumentTypeError): + _single_char_delimiter(raw) + + +# --------------------------------------------------------------------------- +# CSV delimiter integration (end-to-end via CLI) +# --------------------------------------------------------------------------- + + +def test_csv_delimiter_backslash_t_parses_tsv_pandas( + tsv_pair: Callable[[list[list[str]], list[list[str]]], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + rows = [["id", "val"], ["1", "a"], ["2", "b"]] + left, right = tsv_pair(rows, rows) + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "\\t", + ] + ) + assert code == 0, f"expected match, stderr: {err}" + + +def test_csv_delimiter_backslash_t_parses_tsv_polars( + tsv_pair: Callable[[list[list[str]], list[list[str]]], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + rows = [["id", "val"], ["1", "a"], ["2", "b"]] + left, right = tsv_pair(rows, rows) + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--csv-delimiter", + "\\t", + ] + ) + assert code == 0, f"expected match, stderr: {err}" + + +def test_csv_delimiter_backslash_t_detects_mismatch( + tsv_pair: Callable[[list[list[str]], list[list[str]]], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + left_rows = [["id", "val"], ["1", "a"], ["2", "b"]] + right_rows = [["id", "val"], ["1", "a"], ["2", "X"]] + left, right = tsv_pair(left_rows, right_rows) + code, _, _ = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "\\t", + ] + ) + assert code == 1 + + +def test_csv_delimiter_real_tab_also_works( + tsv_pair: Callable[[list[list[str]], list[list[str]]], tuple[Path, Path]], + cli: Callable[[list[str]], tuple[int, str, str]], +) -> None: + """A real tab character passed programmatically should still work.""" + rows = [["id", "val"], ["1", "a"], ["2", "b"]] + left, right = tsv_pair(rows, rows) + code, _, err = cli( + [ + "compare", + "--left", + str(left), + "--right", + str(right), + "--on", + "id", + "--backend", + "pandas", + "--csv-delimiter", + "\t", + ] + ) + assert code == 0, f"expected match, stderr: {err}" + + +# --------------------------------------------------------------------------- +# load_snowflake staging +# --------------------------------------------------------------------------- + +_snowflake_missing = importlib.util.find_spec("snowflake") is None +_skip_no_snowflake = pytest.mark.skipif( + _snowflake_missing, reason="snowflake.snowpark not installed" +) + + +@_skip_no_snowflake +def test_load_snowflake_staging_raises_error_when_no_database( + mock_snowflake_session: MagicMock, + tmp_path: Path, +) -> None: + """load_snowflake with a local CSV and session.get_current_database()=None + must raise BadArgsError, not return a broken '.' -prefixed table ref.""" + mock_snowflake_session.get_current_database.return_value = None + csv_file = tmp_path / "data.csv" + csv_file.write_text("id,val\n1,a\n2,b\n") + with pytest.raises(BadArgsError, match=r"SNOWFLAKE_DATABASE|database"): + load_snowflake(mock_snowflake_session, str(csv_file), "csv") + + +@_skip_no_snowflake +def test_load_snowflake_staging_raises_error_when_no_schema( + mock_snowflake_session: MagicMock, + tmp_path: Path, +) -> None: + """load_snowflake with a local CSV and session.get_current_schema()=None + must raise BadArgsError, not return a broken ref like 'PROD..DATACOMPY_TMP_...'.""" + mock_snowflake_session.get_current_schema.return_value = None + csv_file = tmp_path / "data.csv" + csv_file.write_text("id,val\n1,a\n2,b\n") + with pytest.raises(BadArgsError, match=r"SNOWFLAKE_SCHEMA|schema"): + load_snowflake(mock_snowflake_session, str(csv_file), "csv") + + +@_skip_no_snowflake +def test_load_snowflake_staging_returns_valid_ref_when_both_set( + mock_snowflake_session: MagicMock, + tmp_path: Path, +) -> None: + """When db and schema are both available, the returned ref must be + a fully-qualified 'db.schema.table' string.""" + csv_file = tmp_path / "data.csv" + csv_file.write_text("id,val\n1,a\n2,b\n") + ref = load_snowflake(mock_snowflake_session, str(csv_file), "csv") + parts = ref.split(".") + assert len(parts) == 3, f"expected db.schema.table, got {ref!r}" + assert parts[0] == "PROD" + assert parts[1] == "PUBLIC" + assert parts[2].startswith("DATACOMPY_TMP_") + + +@_skip_no_snowflake +def test_load_snowflake_csv_delimiter_is_honoured_when_staging( + mock_snowflake_session: MagicMock, + tmp_path: Path, +) -> None: + """When a semicolon-delimited CSV is staged, write_pandas must receive + a DataFrame with two separate columns, not a single column whose name + contains the delimiter.""" + semi_csv = tmp_path / "data.csv" + semi_csv.write_text("id;val\n1;a\n2;b\n") + + captured_df: list[pd.DataFrame] = [] + + def _capture_write_pandas(df: pd.DataFrame, **kwargs: object) -> None: + captured_df.append(df) + + mock_snowflake_session.write_pandas.side_effect = _capture_write_pandas + + load_snowflake(mock_snowflake_session, str(semi_csv), "csv", csv_delimiter=";") + + assert len(captured_df) == 1, "write_pandas should have been called once" + df = captured_df[0] + assert list(df.columns) == [ + "id", + "val", + ], f"Expected ['id', 'val'], got {list(df.columns)!r} -- delimiter was ignored" diff --git a/tests/cli/test_parser.py b/tests/cli/test_parser.py new file mode 100644 index 00000000..2347c6e5 --- /dev/null +++ b/tests/cli/test_parser.py @@ -0,0 +1,191 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the CLI argument parser shape and defaults.""" + +import pytest +from datacompy.cli.parser import build_parser + + +def test_compare_defaults() -> None: + p = build_parser() + args = p.parse_args( + ["compare", "--left", "a.csv", "--right", "b.csv", "--on", "id"] + ) + assert args.backend == "polars" + assert args.abs_tol == pytest.approx(0.0) + assert args.rel_tol == pytest.approx(0.0) + assert args.ignore_spaces is False + assert args.ignore_case is False + assert args.ignore_extra_columns is False + assert args.cast_column_names_lower is True + assert args.df1_name is None # resolved to stem in to_compare_args + assert args.df2_name is None # resolved to stem in to_compare_args + assert args.ignore_unique_rows is False + assert args.csv_delimiter == "," + assert args.sample_count == 10 + assert args.column_count == 10 + assert args.max_unequal_rows is None + assert args.json is False + assert args.quiet is False + + +@pytest.mark.parametrize("backend", ["pandas", "polars", "spark", "snowflake"]) +def test_compare_backend_choices(backend: str) -> None: + p = build_parser() + args = p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on", + "id", + "--backend", + backend, + ] + ) + assert args.backend == backend + + +def test_compare_invalid_backend_exits() -> None: + p = build_parser() + with pytest.raises(SystemExit) as exc: + p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on", + "id", + "--backend", + "duckdb", + ] + ) + assert exc.value.code == 2 + + +@pytest.mark.parametrize("fmt", ["csv", "parquet", "json"]) +def test_compare_format_choices(fmt: str) -> None: + p = build_parser() + args = p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on", + "id", + "--format", + fmt, + ] + ) + assert args.format == fmt + + +def test_compare_multi_on_columns() -> None: + p = build_parser() + args = p.parse_args( + ["compare", "--left", "a.csv", "--right", "b.csv", "--on", "id", "--on", "date"] + ) + assert args.on == ["id", "date"] + + +def test_compare_on_index_flag() -> None: + p = build_parser() + args = p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on-index", + "--backend", + "pandas", + ] + ) + assert args.on_index is True + assert args.on is None + + +def test_compare_version(capsys: pytest.CaptureFixture[str]) -> None: + p = build_parser() + with pytest.raises(SystemExit) as exc: + p.parse_args(["--version"]) + assert exc.value.code == 0 + out = capsys.readouterr().out + assert "datacompy" in out + + +def test_csv_delimiter_multi_char_rejected_at_parse_time() -> None: + """--csv-delimiter with more than one character should fail at argparse parse + time (exit 2), not later during validation.""" + p = build_parser() + with pytest.raises(SystemExit) as exc: + p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on", + "id", + "--csv-delimiter", + "abc", + ] + ) + assert exc.value.code == 2 + + +def test_max_unequal_rows_negative_rejected_at_parse_time() -> None: + """--max-unequal-rows with a negative value should fail at argparse parse time.""" + p = build_parser() + with pytest.raises(SystemExit) as exc: + p.parse_args( + [ + "compare", + "--left", + "a.csv", + "--right", + "b.csv", + "--on", + "id", + "--max-unequal-rows", + "-1", + ] + ) + assert exc.value.code == 2 + + +def test_debug_flag_default_false() -> None: + p = build_parser() + args = p.parse_args( + ["compare", "--left", "a.csv", "--right", "b.csv", "--on", "id"] + ) + assert args.debug is False + + +def test_debug_flag_set_true() -> None: + p = build_parser() + args = p.parse_args( + ["--debug", "compare", "--left", "a.csv", "--right", "b.csv", "--on", "id"] + ) + assert args.debug is True