diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62c3a79e..19c1a30d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,8 +12,10 @@ repos: rev: v5.0.0 hooks: - id: trailing-whitespace + exclude: ^tests/snapshots/ - id: debug-statements - id: end-of-file-fixer + exclude: ^tests/snapshots/ - repo: https://github.com/tox-dev/pyproject-fmt rev: "v2.5.0" hooks: diff --git a/README.md b/README.md index 60dd2f76..e796a5c3 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,36 @@ pip install datacompy[snowflake] - Snowflake/Snowpark: ([See documentation](https://capitalone.github.io/datacompy/snowflake_usage.html)) +## Programmatic Report Access + +Every compare object exposes `build_report_data()` which returns a typed +[`ReportData`](https://capitalone.github.io/datacompy/report_api.html) object +— useful for dashboards, JSON export, or custom rendering without relying on +the string report: + +```python +import pandas as pd +from datacompy import PandasCompare + +df1 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]}) +df2 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 99, 30]}) + +compare = PandasCompare(df1, df2, join_columns="id") + +# Access structured data directly +data = compare.build_report_data() +print(data.row_summary.unequal_rows) # 1 +print(data.mismatch_stats.stats[0].column) # 'val' + +# Render / export — methods live on ReportData itself +print(data.render()) # same text as compare.report() +data.save("report.html") # HTML file +data.to_dict() # JSON-serializable dict +``` + +See the [Report API documentation](https://capitalone.github.io/datacompy/report_api.html) for the full reference. + + ## Contributors We welcome and appreciate your contributions! Before we can accept any contributions, we ask that you please be sure to diff --git a/datacompy/__init__.py b/datacompy/__init__.py index e392fccf..490ad120 100644 --- a/datacompy/__init__.py +++ b/datacompy/__init__.py @@ -23,11 +23,27 @@ from datacompy.base import BaseCompare from datacompy.pandas import PandasCompare from datacompy.polars import PolarsCompare +from datacompy.report import ( + ColumnComparison, + ColumnSummary, + MismatchStat, + MismatchStats, + ReportData, + RowSummary, + UniqueRowsData, +) __all__ = [ "BaseCompare", + "ColumnComparison", + "ColumnSummary", + "MismatchStat", + "MismatchStats", "PandasCompare", "PolarsCompare", + "ReportData", + "RowSummary", + "UniqueRowsData", ] try: diff --git a/datacompy/base.py b/datacompy/base.py index 22274c8d..40883e2b 100644 --- a/datacompy/base.py +++ b/datacompy/base.py @@ -25,20 +25,39 @@ from abc import ABC, abstractmethod from collections import Counter from pathlib import Path -from typing import Any, Dict, List +from typing import TYPE_CHECKING, Any, Dict, List, TypedDict from jinja2 import Environment, FileSystemLoader, select_autoescape from ordered_set import OrderedSet +if TYPE_CHECKING: + import datacompy.report + LOG = logging.getLogger(__name__) +class ColumnStat(TypedDict): + """Typed contract for per-column comparison statistics populated by each backend.""" + + column: str + match_column: str + match_cnt: int + unequal_cnt: int + dtype1: str + dtype2: str + all_match: bool + max_diff: float + null_diff: int + rel_tol: float + abs_tol: float + + class BaseCompare(ABC): """Base comparison class.""" @property def df1(self) -> Any: - """Get the first dataframe.""" + """The first dataframe.""" return self._df1 # type: ignore @df1.setter @@ -49,7 +68,7 @@ def df1(self, df1: Any) -> None: @property def df2(self) -> Any: - """Get the second dataframe.""" + """The second dataframe.""" return self._df2 # type: ignore @df2.setter @@ -60,7 +79,7 @@ def df2(self, df2: Any) -> None: @property def sensitive_columns(self) -> List[str] | None: - """Get the list of sensitive columns.""" + """Sensitive columns to mask in report output.""" return self._sensitive_columns def _set_and_validate_sensitive_columns( @@ -191,7 +210,197 @@ def all_mismatch(self, ignore_matching_cols: bool = False) -> Any: """Get all rows that mismatch.""" pass - @abstractmethod + # ------------------------------------------------------------------ + # Backend primitives — override in subclasses where the default + # (.shape / .columns) does not apply (e.g. Spark / Snowflake). + # ------------------------------------------------------------------ + + def _table_shape(self, df: Any) -> tuple[int, int]: + """Return ``(rows, columns)`` for *df*. + + Spark and Snowflake backends override this to call ``.count()``. + """ + return df.shape # type: ignore[no-any-return] + + def _row_count(self, df: Any, _cache: Dict[int, int] | None = None) -> int: + """Return the row count for *df*, using *_cache* when provided. + + Parameters + ---------- + df : + The DataFrame whose rows to count. + _cache : dict, optional + Mapping from ``id(df)`` to cached count. Spark and Snowflake + backends pass this in to avoid redundant ``.count()`` actions. + """ + if _cache is not None: + key = id(df) + if key not in _cache: + _cache[key] = int(df.shape[0]) + return _cache[key] + return int(df.shape[0]) + + def _select_first_n_columns(self, df: Any, n: int) -> Any: + """Return *df* with only its first *n* columns. + + Pandas overrides this with ``.iloc[:, :n]``. + """ + return df.select(list(df.columns)[:n]) + + def _column_names(self, df: Any) -> List[str]: + """Return column names of *df* as a plain list.""" + return list(df.columns) # type: ignore[return-value] + + # ------------------------------------------------------------------ + # Concrete report-building — shared across all backends + # ------------------------------------------------------------------ + + def build_report_data( + self, sample_count: int = 10, column_count: int = 10 + ) -> "datacompy.report.ReportData": + """Build a typed :class:`~datacompy.report.ReportData` from this comparison. + + Parameters + ---------- + sample_count : int, optional + Maximum number of sample rows to include per section. Defaults to 10. + column_count : int, optional + Maximum number of columns to show in unique-row samples. Defaults to 10. + + Returns + ------- + datacompy.report.ReportData + Immutable data object suitable for rendering or programmatic use. + + Examples + -------- + >>> data = compare.build_report_data() + >>> print(data.row_summary.unequal_rows) + """ + from datacompy.report import ( + ColumnComparison, + ColumnSummary, + MismatchStat, + MismatchStats, + ReportData, + RowSummary, + UniqueRowsData, + ) + + # Per-call row-count cache so Spark/Snowflake only trigger .count() once + # per unique DataFrame object. + row_count_cache: Dict[int, int] = {} + + # ---- column summary ------------------------------------------ + df1_unq_cols = self.df1_unq_columns() + df2_unq_cols = self.df2_unq_columns() + column_summary = ColumnSummary( + common_columns=len(self.intersect_columns()), + df1_unique=len(df1_unq_cols), + df1_unique_columns=tuple(df1_unq_cols), + df2_unique=len(df2_unq_cols), + df2_unique_columns=tuple(df2_unq_cols), + df1_name=self.df1_name, + df2_name=self.df2_name, + ) + + # ---- row summary --------------------------------------------- + intersect_count = self._row_count(self.intersect_rows, row_count_cache) + df1_unq_count = self._row_count(self.df1_unq_rows, row_count_cache) + df2_unq_count = self._row_count(self.df2_unq_rows, row_count_cache) + matching_rows = self.count_matching_rows() + + on_index: bool = getattr(self, "on_index", False) + row_summary = RowSummary( + match_columns=tuple(self.join_columns), + on_index=on_index, + has_duplicates=bool(self._any_dupes), + abs_tol=self.abs_tol, + rel_tol=self.rel_tol, + common_rows=intersect_count, + df1_unique=df1_unq_count, + df2_unique=df2_unq_count, + unequal_rows=intersect_count - matching_rows, + equal_rows=matching_rows, + df1_name=self.df1_name, + df2_name=self.df2_name, + ) + + # ---- column comparison --------------------------------------- + column_comparison = ColumnComparison( + unequal_columns=len([c for c in self.column_stats if c["unequal_cnt"] > 0]), + equal_columns=len([c for c in self.column_stats if c["unequal_cnt"] == 0]), + unequal_values=sum(c["unequal_cnt"] for c in self.column_stats), + ) + + # ---- mismatch stats ------------------------------------------ + stat_list: List[MismatchStat] = [] + sample_list: List[Any] = [] + + for col in self.column_stats: + if not col["all_match"]: + stat_list.append( + MismatchStat( + column=col["column"], + dtype1=col["dtype1"], + dtype2=col["dtype2"], + unequal_cnt=col["unequal_cnt"], + max_diff=col["max_diff"], + null_diff=col["null_diff"], + rel_tol=col["rel_tol"], + abs_tol=col["abs_tol"], + ) + ) + if col["unequal_cnt"] > 0: + sample_list.append( + self.sample_mismatch( + col["column"], sample_count, for_display=True + ) + ) + + if stat_list: + mismatch_stats = MismatchStats( + has_mismatches=True, + has_samples=len(sample_list) > 0 and sample_count > 0, + stats=tuple(sorted(stat_list, key=lambda s: s.column)), + samples=tuple(df_to_str(s) for s in sample_list), + df1_name=self.df1_name, + df2_name=self.df2_name, + ) + else: + mismatch_stats = MismatchStats(has_mismatches=False, has_samples=False) + + # ---- unique rows data ---------------------------------------- + def _unique_rows_data(df: Any, unq_count: int) -> "UniqueRowsData": + min_sample = min(sample_count, unq_count) + min_cols = min(column_count, len(self._column_names(df))) + if min_sample > 0: + rows_str = df_to_str( + self._select_first_n_columns(df, min_cols), + sample_count=min_sample, + ) + else: + rows_str = "" + return UniqueRowsData(has_rows=min_sample > 0, rows=rows_str) + + df1_unique_rows = _unique_rows_data(self.df1_unq_rows, df1_unq_count) + df2_unique_rows = _unique_rows_data(self.df2_unq_rows, df2_unq_count) + + # ---- assemble ------------------------------------------------ + return ReportData( + df1_name=self.df1_name, + df2_name=self.df2_name, + df1_shape=self._table_shape(self.df1), + df2_shape=self._table_shape(self.df2), + column_count=column_count, + column_summary=column_summary, + row_summary=row_summary, + column_comparison=column_comparison, + mismatch_stats=mismatch_stats, + df1_unique_rows=df1_unique_rows, + df2_unique_rows=df2_unique_rows, + ) + def report( self, sample_count: int = 10, @@ -201,27 +410,36 @@ def report( ) -> str: """Return a string representation of a report. + The representation can then be printed or saved to a file. You can customize the + report's appearance by providing a custom Jinja2 template. + Parameters ---------- sample_count : int, optional The number of sample records to return. Defaults to 10. - column_count : int, optional The number of columns to display in the sample records output. Defaults to 10. - html_file : str, optional HTML file name to save report output to. If ``None`` the file creation will be skipped. - template_path : str, optional Path to a custom Jinja2 template file to use for report generation. - If ``None``, the default template will be used. + If ``None``, the default template will be used. The template receives the + context variables documented on :class:`~datacompy.report.ReportData`. Returns ------- str The report, formatted according to the template. + + See Also + -------- + build_report_data : Access the structured data without rendering. """ - pass + data = self.build_report_data(sample_count, column_count) + text = data.render(template_path=template_path) + if html_file: + save_html_report(text, html_file) + return text def reveal_sensitive_columns(self) -> None: """Reveals all sensitive columns. @@ -432,7 +650,7 @@ def df_to_str(df: Any, sample_count: int | None = None, on_index: bool = False) if hasattr(df, "to_pandas"): if sample_count is not None and len(df) > sample_count: df = df.head(sample_count) - return df.to_pandas().to_string() + return str(df) # Fallback to str() if we can't determine the type return str(df) diff --git a/datacompy/pandas.py b/datacompy/pandas.py index 496e0d49..029f68fd 100644 --- a/datacompy/pandas.py +++ b/datacompy/pandas.py @@ -29,10 +29,8 @@ from datacompy.base import ( BaseCompare, - df_to_str, + ColumnStat, get_column_tolerance, - render, - save_html_report, temp_column_name, validate_tolerance_parameter, ) @@ -161,7 +159,7 @@ def __init__( self.df1_unq_rows: pd.DataFrame self.df2_unq_rows: pd.DataFrame self.intersect_rows: pd.DataFrame - self.column_stats: List[Dict[str, Any]] = [] + self.column_stats: List[ColumnStat] = [] self._compare(ignore_spaces=ignore_spaces, ignore_case=ignore_case) def _get_comparators(self) -> List[BaseComparator]: @@ -173,7 +171,7 @@ def _get_comparators(self) -> List[BaseComparator]: @property def df1(self) -> pd.DataFrame: - """Get the first dataframe.""" + """The first dataframe.""" return self._df1 @df1.setter @@ -186,7 +184,7 @@ def df1(self, df1: pd.DataFrame) -> None: @property def df2(self) -> pd.DataFrame: - """Get the second dataframe.""" + """The second dataframe.""" return self._df2 @df2.setter @@ -710,279 +708,11 @@ def all_mismatch(self, ignore_matching_cols: bool = False) -> pd.DataFrame: mm_bool = self.intersect_rows[match_list].all(axis="columns") return self.intersect_rows[~mm_bool][self.join_columns + return_list] - def _get_column_summary(self) -> dict: - """Generate column summary data for the report. + def _select_first_n_columns(self, df: Any, n: int) -> Any: + return df.iloc[:, :n] - Returns - ------- - dict - Dictionary containing column summary information. - """ - return { - "column_summary": { - "common_columns": len(self.intersect_columns()), - "df1_unique": len(self.df1_unq_columns()), - "df2_unique": len(self.df2_unq_columns()), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - } - } - - def _get_row_summary(self) -> dict: - """Generate row summary data for the report. - - Returns - ------- - dict - Dictionary containing row summary information. - """ - return { - "row_summary": { - "match_columns": "index" - if self.on_index - else ", ".join(self.join_columns), - "abs_tol": self.abs_tol, - "rel_tol": self.rel_tol, - "common_rows": self.intersect_rows.shape[0], - "df1_unique": self.df1_unq_rows.shape[0], - "df2_unique": self.df2_unq_rows.shape[0], - "unequal_rows": self.intersect_rows.shape[0] - - self.count_matching_rows(), - "equal_rows": self.count_matching_rows(), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "has_duplicates": "Yes" if self._any_dupes else "No", - } - } - - def _get_column_comparison(self) -> dict: - """Generate column comparison statistics for the report. - - Returns - ------- - dict - Dictionary containing column comparison information. - """ - return { - "column_comparison": { - "unequal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] > 0] - ), - "equal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] == 0] - ), - "unequal_values": sum(col["unequal_cnt"] for col in self.column_stats), - } - } - - def _get_mismatch_stats(self, sample_count: int) -> dict: - """Generate mismatch statistics for the report. - - Parameters - ---------- - sample_count : int - Number of samples to include in the report. - - Returns - ------- - dict - Dictionary containing mismatch statistics. - """ - mismatch_stats = [] - match_sample = [] - any_mismatch = False - - for column in self.column_stats: - if not column["all_match"]: - any_mismatch = True - mismatch_stats.append( - { - "column": column["column"], - "dtype1": column["dtype1"], - "dtype2": column["dtype2"], - "unequal_cnt": column["unequal_cnt"], - "max_diff": column["max_diff"], - "null_diff": column["null_diff"], - "rel_tol": column["rel_tol"], - "abs_tol": column["abs_tol"], - } - ) - if column["unequal_cnt"] > 0: - match_sample.append( - self.sample_mismatch( - column["column"], sample_count, for_display=True - ) - ) - - if any_mismatch: - return { - "mismatch_stats": { - "has_mismatches": True, - "stats": sorted(mismatch_stats, key=lambda x: x["column"]), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "samples": [df_to_str(sample) for sample in match_sample], - "has_samples": len(match_sample) > 0 and sample_count > 0, - } - } - return { - "mismatch_stats": { - "has_mismatches": False, - "has_samples": False, - } - } - - def _get_unique_rows_data(self, sample_count: int, column_count: int) -> dict: - """Generate data for unique rows in both dataframes. - - Parameters - ---------- - sample_count : int - Number of samples to include. - column_count : int - Number of columns to include. - - Returns - ------- - dict - Dictionary containing unique rows data for both dataframes. - """ - min_sample_count_df1 = min(sample_count, self.df1_unq_rows.shape[0]) - min_sample_count_df2 = min(sample_count, self.df2_unq_rows.shape[0]) - min_column_count_df1 = min(column_count, self.df1_unq_rows.shape[1]) - min_column_count_df2 = min(column_count, self.df2_unq_rows.shape[1]) - - return { - "df1_unique_rows": { - "has_rows": min_sample_count_df1 > 0, - "rows": df_to_str( - self.df1_unq_rows.iloc[:, :min_column_count_df1], - sample_count=min_sample_count_df1, - ) - if self.df1_unq_rows.shape[0] > 0 - else "", - "columns": list(self.df1_unq_rows.columns[:min_column_count_df1]) - if self.df1_unq_rows.shape[0] > 0 - else "", - }, - "df2_unique_rows": { - "has_rows": min_sample_count_df2 > 0, - "rows": df_to_str( - self.df2_unq_rows.iloc[:, :min_column_count_df2], - sample_count=min_sample_count_df2, - ) - if self.df2_unq_rows.shape[0] > 0 - else "", - "columns": list(self.df2_unq_rows.columns[:min_column_count_df2]) - if self.df2_unq_rows.shape[0] > 0 - else "", - }, - } - - def report( - self, - sample_count: int = 10, - column_count: int = 10, - html_file: str | None = None, - template_path: str | None = None, - ) -> str: - """Return a string representation of a report. - - The representation can then be printed or saved to a file. You can customize the - report's appearance by providing a custom Jinja2 template. - - Parameters - ---------- - sample_count : int, optional - The number of sample records to return. Defaults to 10. - - column_count : int, optional - The number of columns to display in the sample records output. Defaults to 10. - - html_file : str, optional - HTML file name to save report output to. If ``None`` the file creation will be skipped. - - template_path : str, optional - Path to a custom Jinja2 template file to use for report generation. - If ``None``, the default template will be used. The template receives the - following context variables: - - - ``column_summary``: Dict with column statistics including: ``common_columns``, ``df1_unique``, ``df2_unique``, ``df1_name``, ``df2_name`` - - ``row_summary``: Dict with row statistics including: ``match_columns``, ``equal_rows``, ``unequal_rows`` - - ``column_comparison``: Dict with column comparison statistics including: ``unequal_columns``, ``equal_columns``, ``unequal_values`` - - ``mismatch_stats``: Dict containing: - - ``stats``: List of dicts with column mismatch statistics (column, match, mismatch, null_diff, etc.) - - ``samples``: Sample rows with mismatched values - - ``has_samples``: Boolean indicating if there are any mismatch samples - - ``has_mismatches``: Boolean indicating if there are any mismatches - - ``df1_unique_rows``: Dict with unique rows in df1 including: ``has_rows``, ``rows``, ``columns`` - - ``df2_unique_rows``: Dict with unique rows in df2 including: ``has_rows``, ``rows``, ``columns`` - - Returns - ------- - str - The report, formatted according to the template. - - Examples - -------- - Basic usage with default template: - - >>> compare = datacompy.Compare(df1, df2, join_columns=['id']) - >>> report = compare.report() - >>> print(report) - - Using a custom template file: - - >>> # Create a custom template file (custom_report.j2) - >>> with open('custom_report.j2', 'w') as f: - ... f.write(''' - ... Comparison Report - ... ================ - ... - ... DataFrames: {{ df1_name }} vs {{ df2_name }} - ... - ... Shape Summary: - ... - {{ df1_name }}: {{ df1_shape[0] }} rows x {{ df1_shape[1] }} columns - ... - {{ df2_name }}: {{ df2_shape[0] }} rows x {{ df2_shape[1] }} columns - ... - ... {% if mismatch_stats %} - ... Mismatched Columns ({{ mismatch_stats|length }}): - ... {% for col in mismatch_stats %} - ... - {{ col.column }} ({{ col.unequal_cnt }} mismatches) - ... {% endfor %} - ... {% else %} - ... No mismatches found in any columns! - ... {% endif %} - ... ''') - ... - >>> # Generate report with custom template - >>> report = compare.report(template_path='custom_report.j2') - >>> print(report) - """ - # Prepare the template data by combining all sections - template_data = { - **self._get_column_summary(), - **self._get_row_summary(), - **self._get_column_comparison(), - **self._get_mismatch_stats(sample_count), - **self._get_unique_rows_data(sample_count, column_count), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "df1_shape": self.df1.shape, - "df2_shape": self.df2.shape, - "column_count": column_count, - } - - # Determine which template to use - template_name = template_path if template_path else "report_template.j2" - - # Render the main report - report = render(template_name, **template_data) - - if html_file: - save_html_report(report, html_file) - - return report + def _column_names(self, df: Any) -> List[str]: + return list(df.columns) def columns_equal( diff --git a/datacompy/polars.py b/datacompy/polars.py index 2319e35f..fcaa445d 100644 --- a/datacompy/polars.py +++ b/datacompy/polars.py @@ -30,10 +30,8 @@ from datacompy.base import ( BaseCompare, - df_to_str, + ColumnStat, get_column_tolerance, - render, - save_html_report, temp_column_name, validate_tolerance_parameter, ) @@ -149,7 +147,7 @@ def __init__( self.df1_unq_rows: pl.DataFrame self.df2_unq_rows: pl.DataFrame self.intersect_rows: pl.DataFrame - self.column_stats: List[Dict[str, Any]] = [] + self.column_stats: List[ColumnStat] = [] self._compare(ignore_spaces=ignore_spaces, ignore_case=ignore_case) def _get_comparators(self) -> List[BaseComparator]: @@ -161,7 +159,7 @@ def _get_comparators(self) -> List[BaseComparator]: @property def df1(self) -> pl.DataFrame: - """Get the first dataframe.""" + """The first dataframe.""" return self._df1 @df1.setter @@ -174,7 +172,7 @@ def df1(self, df1: pl.DataFrame) -> None: @property def df2(self) -> pl.DataFrame: - """Get the second dataframe.""" + """The second dataframe.""" return self._df2 @df2.setter @@ -419,8 +417,8 @@ def _intersect_compare(self, ignore_spaces: bool, ignore_case: bool) -> None: creates a column column_match which is True for matches, False otherwise. """ - match_cnt: int | float - null_diff: int | float + match_cnt: int + null_diff: int LOG.debug("Comparing intersection") for column in self.intersect_columns(): @@ -453,14 +451,16 @@ def _intersect_compare(self, ignore_spaces: bool, ignore_case: bool) -> None: comparators=self._get_comparators(), ).alias(col_match) ) - match_cnt = self.intersect_rows[col_match].sum() + match_cnt = int(self.intersect_rows[col_match].sum()) max_diff = calculate_max_diff( self.intersect_rows[col_1], self.intersect_rows[col_2] ) - null_diff = ( - (self.intersect_rows[col_1].is_null()) - ^ (self.intersect_rows[col_2].is_null()) - ).sum() + null_diff = int( + ( + (self.intersect_rows[col_1].is_null()) + ^ (self.intersect_rows[col_2].is_null()) + ).sum() + ) if row_cnt > 0: match_rate = float(match_cnt) / row_cnt else: @@ -703,246 +703,6 @@ def all_mismatch(self, ignore_matching_cols: bool = False) -> pl.DataFrame: .select(self.join_columns + return_list) ) - def _get_column_summary(self) -> dict: - """Generate column summary data for the report. - - Returns - ------- - dict - Dictionary containing column summary information. - """ - return { - "column_summary": { - "common_columns": len(self.intersect_columns()), - "df1_unique": f"{len(self.df1_unq_columns())} {self.df1_unq_columns().items}", - "df2_unique": f"{len(self.df2_unq_columns())} {self.df2_unq_columns().items}", - "df1_name": self.df1_name, - "df2_name": self.df2_name, - } - } - - def _get_row_summary(self) -> dict: - """Generate row summary data for the report. - - Returns - ------- - dict - Dictionary containing row summary information. - """ - return { - "row_summary": { - "match_columns": ", ".join(self.join_columns), - "abs_tol": self.abs_tol, - "rel_tol": self.rel_tol, - "common_rows": self.intersect_rows.shape[0], - "df1_unique": self.df1_unq_rows.shape[0], - "df2_unique": self.df2_unq_rows.shape[0], - "unequal_rows": self.intersect_rows.shape[0] - - self.count_matching_rows(), - "equal_rows": self.count_matching_rows(), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "has_duplicates": "Yes" if self._any_dupes else "No", - } - } - - def _get_column_comparison(self) -> dict: - """Generate column comparison statistics for the report. - - Returns - ------- - dict - Dictionary containing column comparison information. - """ - return { - "column_comparison": { - "unequal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] > 0] - ), - "equal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] == 0] - ), - "unequal_values": sum(col["unequal_cnt"] for col in self.column_stats), - } - } - - def _get_mismatch_stats(self, sample_count: int) -> dict: - """Generate mismatch statistics for the report. - - Parameters - ---------- - sample_count : int - Number of samples to include in the report. - - Returns - ------- - dict - Dictionary containing mismatch statistics. - """ - match_stats = [] - match_sample = [] - any_mismatch = False - - for column in self.column_stats: - if not column["all_match"]: - any_mismatch = True - match_stats.append( - { - "column": column["column"], - "dtype1": column["dtype1"], - "dtype2": column["dtype2"], - "unequal_cnt": column["unequal_cnt"], - "max_diff": column["max_diff"], - "null_diff": column["null_diff"], - "rel_tol": column["rel_tol"], - "abs_tol": column["abs_tol"], - } - ) - if column["unequal_cnt"] > 0: - match_sample.append( - self.sample_mismatch( - column["column"], sample_count, for_display=True - ) - ) - - if any_mismatch: - return { - "mismatch_stats": { - "has_mismatches": True, - "stats": match_stats, - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "samples": [df_to_str(sample) for sample in match_sample], - "has_samples": len(match_sample) > 0 and sample_count > 0, - } - } - return { - "mismatch_stats": { - "has_mismatches": False, - "has_samples": False, - } - } - - def _get_unique_rows_data(self, sample_count: int, column_count: int) -> dict: - """Generate data for unique rows in both dataframes. - - Parameters - ---------- - sample_count : int - Number of samples to include. - column_count : int - Number of columns to include. - - Returns - ------- - dict - Dictionary containing unique rows data for both dataframes. - """ - min_sample_count_df1 = min(sample_count, self.df1_unq_rows.shape[0]) - min_sample_count_df2 = min(sample_count, self.df2_unq_rows.shape[0]) - min_column_count_df1 = min(column_count, self.df1_unq_rows.shape[1]) - min_column_count_df2 = min(column_count, self.df2_unq_rows.shape[1]) - - return { - "df1_unique_rows": { - "has_rows": min_sample_count_df1 > 0, - "rows": df_to_str( - self.df1_unq_rows.select( - self.df1_unq_rows.columns[:min_column_count_df1] - ), - sample_count=min_sample_count_df1, - ) - if self.df1_unq_rows.shape[0] > 0 - else "", - "columns": list(self.df1_unq_rows.columns[:min_column_count_df1]) - if self.df1_unq_rows.shape[0] > 0 - else "", - }, - "df2_unique_rows": { - "has_rows": min_sample_count_df2 > 0, - "rows": df_to_str( - self.df2_unq_rows.select( - self.df2_unq_rows.columns[:min_column_count_df2] - ), - sample_count=min_sample_count_df2, - ) - if self.df2_unq_rows.shape[0] > 0 - else "", - "columns": list(self.df2_unq_rows.columns[:min_column_count_df2]) - if self.df2_unq_rows.shape[0] > 0 - else "", - }, - } - - def report( - self, - sample_count: int = 10, - column_count: int = 10, - html_file: str | None = None, - template_path: str | None = None, - ) -> str: - """Return a string representation of a report. - - The representation can then be printed or saved to a file. You can customize the - report's appearance by providing a custom Jinja2 template. - - Parameters - ---------- - sample_count : int, optional - The number of sample records to return. Defaults to 10. - - column_count : int, optional - The number of columns to display in the sample records output. Defaults to 10. - - html_file : str, optional - HTML file name to save report output to. If ``None`` the file creation will be skipped. - - template_path : str, optional - Path to a custom Jinja2 template file to use for report generation. - If ``None``, the default template will be used. The template receives the - following context variables: - - - ``column_summary``: Dict with column statistics including: ``common_columns``, ``df1_unique``, ``df2_unique``, ``df1_name``, ``df2_name`` - - ``row_summary``: Dict with row statistics including: ``match_columns``, ``equal_rows``, ``unequal_rows`` - - ``column_comparison``: Dict with column comparison statistics including: ``unequal_columns``, ``equal_columns``, ``unequal_values`` - - ``mismatch_stats``: Dict containing: - - ``stats``: List of dicts with column mismatch statistics (column, match, mismatch, null_diff, etc.) - - ``samples``: Sample rows with mismatched values - - ``has_samples``: Boolean indicating if there are any mismatch samples - - ``has_mismatches``: Boolean indicating if there are any mismatches - - ``df1_unique_rows``: Dict with unique rows in df1 including: ``has_rows``, ``rows``, ``columns`` - - ``df2_unique_rows``: Dict with unique rows in df2 including: ``has_rows``, ``rows``, ``columns`` - - Returns - ------- - str - The report, formatted according to the template. - """ - # Prepare template data - template_data = { - **self._get_column_summary(), - **self._get_row_summary(), - **self._get_column_comparison(), - **self._get_mismatch_stats(sample_count), - **self._get_unique_rows_data(sample_count, column_count), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "df1_shape": (self.df1.shape[0], self.df1.shape[1]), - "df2_shape": (self.df2.shape[0], self.df2.shape[1]), - "column_count": column_count, - } - - # Determine which template to use - template_name = template_path if template_path else "report_template.j2" - - # Render the main report - report = render(template_name, **template_data) - - if html_file: - save_html_report(report, html_file) - - return report - def columns_equal( col_1: pl.Series, diff --git a/datacompy/report.py b/datacompy/report.py new file mode 100644 index 00000000..093110f7 --- /dev/null +++ b/datacompy/report.py @@ -0,0 +1,331 @@ +# +# Copyright 2026 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Typed data model for DataComPy report generation. + +All backends produce a :class:`~datacompy.report.ReportData` instance via +``compare.build_report_data()``. :class:`~datacompy.report.ReportData` owns rendering to +text and HTML, and exposes ``to_dict()`` for programmatic consumers. + +Examples +-------- +Programmatic access without rendering: + +>>> data = compare.build_report_data() +>>> print(data.row_summary.unequal_rows) +>>> print(data.mismatch_stats.stats[0].column) + +Render to text and save HTML: + +>>> data = compare.build_report_data() +>>> print(data.render()) +>>> data.save("comparison.html") + +Export as JSON-serializable dict: + +>>> import json +>>> json.dumps(data.to_dict()) +""" + +import dataclasses +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Tuple + +from datacompy.base import render, save_html_report + + +@dataclass(frozen=True) +class ColumnSummary: + """Summary of column overlap between the two DataFrames. + + Attributes + ---------- + common_columns : int + Number of columns present in both DataFrames. + df1_unique : int + Number of columns only in df1. + df1_unique_columns : tuple of str + Names of columns only in df1. + df2_unique : int + Number of columns only in df2. + df2_unique_columns : tuple of str + Names of columns only in df2. + df1_name : str + Label for the first DataFrame. + df2_name : str + Label for the second DataFrame. + """ + + common_columns: int + df1_unique: int + df1_unique_columns: Tuple[str, ...] + df2_unique: int + df2_unique_columns: Tuple[str, ...] + df1_name: str + df2_name: str + + +@dataclass(frozen=True) +class RowSummary: + """Summary of row overlap and match statistics. + + Attributes + ---------- + match_columns : tuple of str + Column(s) used to join/match rows. Empty tuple when matching on index. + on_index : bool + True when the comparison was joined on DataFrame index rather than columns. + has_duplicates : bool + Whether duplicate join-key values were found. + abs_tol : float or dict + Absolute tolerance used for numeric comparisons. + rel_tol : float or dict + Relative tolerance used for numeric comparisons. + common_rows : int + Number of rows present in both DataFrames after joining. + df1_unique : int + Number of rows only in df1. + df2_unique : int + Number of rows only in df2. + unequal_rows : int + Number of common rows where at least one compared column differs. + equal_rows : int + Number of common rows where all compared columns match. + df1_name : str + Label for the first DataFrame. + df2_name : str + Label for the second DataFrame. + """ + + match_columns: Tuple[str, ...] + on_index: bool + has_duplicates: bool + abs_tol: Any + rel_tol: Any + common_rows: int + df1_unique: int + df2_unique: int + unequal_rows: int + equal_rows: int + df1_name: str + df2_name: str + + +@dataclass(frozen=True) +class ColumnComparison: + """Aggregate column comparison counts. + + Attributes + ---------- + unequal_columns : int + Number of compared columns with at least one unequal value. + equal_columns : int + Number of compared columns where all values matched. + unequal_values : int + Total number of individual cell values that did not match. + """ + + unequal_columns: int + equal_columns: int + unequal_values: int + + +@dataclass(frozen=True) +class MismatchStat: + """Per-column mismatch statistics. + + Attributes + ---------- + column : str + Column name. + dtype1 : str + Data type in df1. + dtype2 : str + Data type in df2. + unequal_cnt : int + Number of rows where the values differ. + max_diff : float + Maximum absolute numeric difference (0.0 for non-numeric columns). + null_diff : int + Number of rows where one value is null and the other is not. + rel_tol : float + Relative tolerance applied to this column. + abs_tol : float + Absolute tolerance applied to this column. + """ + + column: str + dtype1: str + dtype2: str + unequal_cnt: int + max_diff: float + null_diff: int + rel_tol: float + abs_tol: float + + +@dataclass(frozen=True) +class MismatchStats: + """Mismatch statistics across all compared columns. + + Attributes + ---------- + has_mismatches : bool + True when at least one column has unequal values or types. + has_samples : bool + True when sample rows are available for display. + stats : tuple of MismatchStat + Per-column statistics, sorted by column name. + samples : tuple of str + Pre-rendered string tables of sample mismatched rows (one per column). + df1_name : str + Label for the first DataFrame. + df2_name : str + Label for the second DataFrame. + """ + + has_mismatches: bool + has_samples: bool + stats: Tuple[MismatchStat, ...] = () + samples: Tuple[str, ...] = () + df1_name: str = "" + df2_name: str = "" + + +@dataclass(frozen=True) +class UniqueRowsData: + """Sample rows that exist in only one of the two DataFrames. + + Attributes + ---------- + has_rows : bool + True when there is at least one unique row to display. + rows : str + Pre-rendered string table of sample rows. + """ + + has_rows: bool + rows: str = "" + + +@dataclass(frozen=True, repr=False) +class ReportData: + """Complete data model for a DataComPy comparison report. + + Produced by ``compare.build_report_data()``. All fields are immutable + so the object can be safely cached or passed across threads. + + Attributes + ---------- + df1_name : str + Label for the first DataFrame. + df2_name : str + Label for the second DataFrame. + df1_shape : tuple of (int, int) + ``(rows, columns)`` of df1. + df2_shape : tuple of (int, int) + ``(rows, columns)`` of df2. + column_count : int + Maximum number of columns displayed in unique-row samples. + column_summary : ColumnSummary + row_summary : RowSummary + column_comparison : ColumnComparison + mismatch_stats : MismatchStats + df1_unique_rows : UniqueRowsData + df2_unique_rows : UniqueRowsData + """ + + df1_name: str + df2_name: str + df1_shape: Tuple[int, int] + df2_shape: Tuple[int, int] + column_count: int + column_summary: ColumnSummary + row_summary: RowSummary + column_comparison: ColumnComparison + mismatch_stats: MismatchStats + df1_unique_rows: UniqueRowsData + df2_unique_rows: UniqueRowsData + + def render(self, template_path: str | None = None) -> str: + """Render the report to a text string. + + Parameters + ---------- + template_path : str, optional + Path to a custom Jinja2 template. When ``None`` the default + ``report_template.j2`` is used. + + Returns + ------- + str + Formatted report text. + """ + return render(template_path or "report_template.j2", **dataclasses.asdict(self)) + + def to_html(self, template_path: str | None = None) -> str: + """Return the report wrapped in a minimal HTML page. + + Parameters + ---------- + template_path : str, optional + Path to a custom Jinja2 template. When ``None`` the default + ``report_template.j2`` is used. + + Returns + ------- + str + HTML string with the text report inside a ``
`` block.
+        """
+        text = self.render(template_path)
+        return (
+            f"DataComPy Report"
+            f"
{text}
" + ) + + def save(self, path: str | Path, template_path: str | None = None) -> None: + """Save the report as an HTML file. + + Parameters + ---------- + path : str or Path + Destination file path. Parent directories are created if needed. + template_path : str, optional + Path to a custom Jinja2 template. When ``None`` the default + ``report_template.j2`` is used. + """ + save_html_report(self.render(template_path), path) + + def to_dict(self) -> Dict[str, Any]: + """Return the report data as a JSON-serializable dict. + + Returns + ------- + dict + ``dataclasses.asdict(self)`` — safe for ``json.dumps()``. + """ + return dataclasses.asdict(self) + + def __str__(self) -> str: + """Return the rendered report as a string.""" + return self.render() + + def __repr__(self) -> str: + """Return a concise developer representation.""" + return ( + f"ReportData(df1={self.df1_name!r}, df2={self.df2_name!r}, " + f"shape1={self.df1_shape}, shape2={self.df2_shape})" + ) diff --git a/datacompy/snowflake.py b/datacompy/snowflake.py index 574b6d5a..fa98bdd2 100644 --- a/datacompy/snowflake.py +++ b/datacompy/snowflake.py @@ -58,10 +58,8 @@ from datacompy.base import ( BaseCompare, - df_to_str, + ColumnStat, get_column_tolerance, - render, - save_html_report, validate_tolerance_parameter, ) from datacompy.comparator import ( @@ -180,7 +178,7 @@ def __init__( self.df1_unq_rows: sp.DataFrame self.df2_unq_rows: sp.DataFrame self.intersect_rows: sp.DataFrame - self.column_stats: List[Dict[str, Any]] = [] + self.column_stats: List[ColumnStat] = [] self._compare(ignore_spaces=ignore_spaces, ignore_case=ignore_case) def _get_comparators(self) -> List[BaseComparator]: @@ -192,7 +190,7 @@ def _get_comparators(self) -> List[BaseComparator]: @property def df1(self) -> "sp.DataFrame": - """Get the first dataframe.""" + """The first dataframe.""" return self._df1 @df1.setter @@ -213,7 +211,7 @@ def df1(self, df1: tuple[Union[str, "sp.DataFrame"], str | None]) -> None: @property def df2(self) -> "sp.DataFrame": - """Get the second dataframe.""" + """The second dataframe.""" return self._df2 @df2.setter @@ -903,256 +901,22 @@ def all_mismatch(self, ignore_matching_cols: bool = False) -> "sp.DataFrame": return mm_rows.select(self.join_columns + return_list) - def _get_column_summary(self) -> dict: - """Generate column summary data for the report. + def _table_shape(self, df: Any) -> tuple[int, int]: + return (df.count(), len(df.columns)) - Returns - ------- - dict - Dictionary containing column summary information. - """ - return { - "column_summary": { - "common_columns": len(self.intersect_columns()), - "df1_unique": f"{len(self.df1_unq_columns())} {self.df1_unq_columns().items}", - "df2_unique": f"{len(self.df2_unq_columns())} {self.df2_unq_columns().items}", - "df1_name": self.df1_name, - "df2_name": self.df2_name, - } - } + def _row_count(self, df: Any, _cache: Dict[int, int] | None = None) -> int: + if _cache is not None: + key = id(df) + if key not in _cache: + _cache[key] = df.count() + return _cache[key] + return int(df.count()) - def _get_row_summary(self) -> dict: - """Generate row summary data for the report. + def _select_first_n_columns(self, df: Any, n: int) -> Any: + return df.select(df.columns[:n]) - Returns - ------- - dict - Dictionary containing row summary information. - """ - intersect_count = self.intersect_rows.count() - df1_unq_count = self.df1_unq_rows.count() - df2_unq_count = self.df2_unq_rows.count() - matching_rows = self.count_matching_rows() - - return { - "row_summary": { - "match_columns": ", ".join(self.join_columns), - "abs_tol": self.abs_tol, - "rel_tol": self.rel_tol, - "common_rows": intersect_count, - "df1_unique": df1_unq_count, - "df2_unique": df2_unq_count, - "unequal_rows": intersect_count - matching_rows, - "equal_rows": matching_rows, - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "has_duplicates": "Yes" if self._any_dupes else "No", - } - } - - def _get_column_comparison(self) -> dict: - """Generate column comparison statistics for the report. - - Returns - ------- - dict - Dictionary containing column comparison information. - """ - return { - "column_comparison": { - "unequal_columns": len( - [c for c in self.column_stats if c["unequal_cnt"] > 0] - ), - "equal_columns": len( - [c for c in self.column_stats if c["unequal_cnt"] == 0] - ), - "unequal_values": sum(c["unequal_cnt"] for c in self.column_stats), - } - } - - def _get_mismatch_stats(self, sample_count: int) -> dict: - """Generate mismatch statistics for the report. - - Parameters - ---------- - sample_count : int - Number of samples to include in the report. - - Returns - ------- - dict - Dictionary containing mismatch statistics. - """ - match_stats = [] - match_sample = [] - any_mismatch = False - - for column in self.column_stats: - if not column["all_match"]: - any_mismatch = True - match_stats.append( - { - "column": column["column"], - "dtype1": column["dtype1"], - "dtype2": column["dtype2"], - "unequal_cnt": column["unequal_cnt"], - "max_diff": column["max_diff"], - "null_diff": column["null_diff"], - "rel_tol": column["rel_tol"], - "abs_tol": column["abs_tol"], - } - ) - if column["unequal_cnt"] > 0: - match_sample.append( - self.sample_mismatch( - column["column"], sample_count, for_display=True - ) - ) - - if any_mismatch: - return { - "mismatch_stats": { - "has_mismatches": True, - "stats": match_stats, - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "samples": [df_to_str(sample) for sample in match_sample], - "has_samples": len(match_sample) > 0 and sample_count > 0, - } - } - return { - "mismatch_stats": { - "has_mismatches": False, - "has_samples": False, - } - } - - def _get_unique_rows_data(self, sample_count: int, column_count: int) -> dict: - """Generate data for unique rows in both dataframes. - - Parameters - ---------- - sample_count : int - Number of samples to include. - column_count : int - Number of columns to include. - - Returns - ------- - dict - Dictionary containing unique rows data for both dataframes. - """ - df1_unq_count = self.df1_unq_rows.count() - df2_unq_count = self.df2_unq_rows.count() - - min_sample_count_df1 = min(sample_count, df1_unq_count) - min_sample_count_df2 = min(sample_count, df2_unq_count) - min_column_count_df1 = min(column_count, len(self.df1_unq_rows.columns)) - min_column_count_df2 = min(column_count, len(self.df2_unq_rows.columns)) - - return { - "df1_unique_rows": { - "has_rows": min_sample_count_df1 > 0, - "rows": df_to_str( - self.df1_unq_rows.select( - self.df1_unq_rows.columns[:min_column_count_df1] - ), - sample_count=min_sample_count_df1, - ) - if df1_unq_count > 0 - else "", - "columns": list(self.df1_unq_rows.columns[:min_column_count_df1]) - if df1_unq_count > 0 - else "", - }, - "df2_unique_rows": { - "has_rows": min_sample_count_df2 > 0, - "rows": df_to_str( - self.df2_unq_rows.select( - self.df2_unq_rows.columns[:min_column_count_df2] - ), - sample_count=min_sample_count_df2, - ) - if df2_unq_count > 0 - else "", - "columns": list(self.df2_unq_rows.columns[:min_column_count_df2]) - if df2_unq_count > 0 - else "", - }, - } - - def report( - self, - sample_count: int = 10, - column_count: int = 10, - html_file: str | None = None, - template_path: str | None = None, - ) -> str: - """Return a string representation of a report. - - The representation can then be printed or saved to a file. You can customize the - report's appearance by providing a custom Jinja2 template. - - Parameters - ---------- - sample_count : int, optional - The number of sample records to return. Defaults to 10. - - column_count : int, optional - The number of columns to display in the sample records output. Defaults to 10. - - html_file : str, optional - HTML file name to save report output to. If ``None`` the file creation will be skipped. - - template_path : str, optional - Path to a custom Jinja2 template file to use for report generation. - If ``None``, the default template will be used. The template receives the - following context variables: - - - ``column_summary``: Dict with column statistics including: ``common_columns``, ``df1_unique``, ``df2_unique``, ``df1_name``, ``df2_name`` - - ``row_summary``: Dict with row statistics including: ``match_columns``, ``equal_rows``, ``unequal_rows`` - - ``column_comparison``: Dict with column comparison statistics including: ``unequal_columns``, ``equal_columns``, ``unequal_values`` - - ``mismatch_stats``: Dict containing: - - ``stats``: List of dicts with column mismatch statistics (column, match, mismatch, null_diff, etc.) - - ``samples``: Sample rows with mismatched values - - ``has_samples``: Boolean indicating if there are any mismatch samples - - ``has_mismatches``: Boolean indicating if there are any mismatches - - ``df1_unique_rows``: Dict with unique rows in df1 including: ``has_rows``, ``rows``, ``columns`` - - ``df2_unique_rows``: Dict with unique rows in df2 including: ``has_rows``, ``rows``, ``columns`` - - Returns - ------- - str - The report, formatted according to the template. - """ - # Get counts for the dataframes - df1_count = self.df1.count() - df2_count = self.df2.count() - - # Prepare template data - template_data = { - **self._get_column_summary(), - **self._get_row_summary(), - **self._get_column_comparison(), - **self._get_mismatch_stats(sample_count), - **self._get_unique_rows_data(sample_count, column_count), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "df1_shape": (df1_count, len(self.df1.columns)), - "df2_shape": (df2_count, len(self.df2.columns)), - "column_count": column_count, - } - - # Determine which template to use - template_name = template_path if template_path else "report_template.j2" - - # Render the main report - report = render(template_name, **template_data) - - if html_file: - save_html_report(report, html_file) - - return report + def _column_names(self, df: Any) -> List[str]: + return list(df.columns) def columns_equal( diff --git a/datacompy/spark.py b/datacompy/spark.py index 5812dd1f..12b30184 100644 --- a/datacompy/spark.py +++ b/datacompy/spark.py @@ -34,10 +34,8 @@ from datacompy.base import ( BaseCompare, - df_to_str, + ColumnStat, get_column_tolerance, - render, - save_html_report, temp_column_name, validate_tolerance_parameter, ) @@ -185,7 +183,7 @@ def __init__( self.df1_unq_rows: pyspark.sql.DataFrame self.df2_unq_rows: pyspark.sql.DataFrame self.intersect_rows: pyspark.sql.DataFrame - self.column_stats: List = [] + self.column_stats: List[ColumnStat] = [] self._compare(ignore_spaces=ignore_spaces, ignore_case=ignore_case) def _get_comparators(self) -> List[BaseComparator]: @@ -197,7 +195,7 @@ def _get_comparators(self) -> List[BaseComparator]: @property def df1(self) -> "pyspark.sql.DataFrame": - """Get the first dataframe.""" + """The first dataframe.""" return self._df1 @df1.setter @@ -210,7 +208,7 @@ def df1(self, df1: "pyspark.sql.DataFrame") -> None: @property def df2(self) -> "pyspark.sql.DataFrame": - """Get the second dataframe.""" + """The second dataframe.""" return self._df2 @df2.setter @@ -866,266 +864,22 @@ def all_mismatch( return mm_rows.select(self.join_columns + return_list) - def _get_column_summary(self) -> dict: - """Generate column summary data for the report. + def _table_shape(self, df: Any) -> tuple[int, int]: + return (df.count(), len(df.columns)) - Returns - ------- - dict - Dictionary containing column summary information. - """ - return { - "column_summary": { - "common_columns": len(self.intersect_columns()), - "df1_unique": f"{len(self.df1_unq_columns())} {self.df1_unq_columns().items}", - "df2_unique": f"{len(self.df2_unq_columns())} {self.df2_unq_columns().items}", - "df1_name": self.df1_name, - "df2_name": self.df2_name, - } - } - - def _get_row_summary(self) -> dict: - """Generate row summary data for the report. + def _row_count(self, df: Any, _cache: Dict[int, int] | None = None) -> int: + if _cache is not None: + key = id(df) + if key not in _cache: + _cache[key] = df.count() + return _cache[key] + return int(df.count()) - Returns - ------- - dict - Dictionary containing row summary information. - """ - intersect_count = self.intersect_rows.count() - df1_unq_count = self.df1_unq_rows.count() - df2_unq_count = self.df2_unq_rows.count() - matching_rows = self.count_matching_rows() - - return { - "row_summary": { - "match_columns": ", ".join(self.join_columns), - "abs_tol": self.abs_tol, - "rel_tol": self.rel_tol, - "common_rows": intersect_count, - "df1_unique": df1_unq_count, - "df2_unique": df2_unq_count, - "unequal_rows": intersect_count - matching_rows, - "equal_rows": matching_rows, - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "has_duplicates": "Yes" if self._any_dupes else "No", - } - } - - def _get_column_comparison(self) -> dict: - """Generate column comparison statistics for the report. + def _select_first_n_columns(self, df: Any, n: int) -> Any: + return df.select(df.columns[:n]) - Returns - ------- - dict - Dictionary containing column comparison information. - """ - return { - "column_comparison": { - "unequal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] > 0] - ), - "equal_columns": len( - [col for col in self.column_stats if col["unequal_cnt"] == 0] - ), - "unequal_values": sum(col["unequal_cnt"] for col in self.column_stats), - } - } - - def _get_mismatch_stats(self, sample_count: int) -> dict: - """Generate mismatch statistics for the report. - - Parameters - ---------- - sample_count : int - Number of samples to include in the report. - - Returns - ------- - dict - Dictionary containing mismatch statistics. - """ - match_stats = [] - match_sample = [] - any_mismatch = False - - for column in self.column_stats: - if not column["all_match"]: - any_mismatch = True - match_stats.append( - { - "column": column["column"], - "dtype1": column["dtype1"], - "dtype2": column["dtype2"], - "unequal_cnt": column["unequal_cnt"], - "max_diff": column["max_diff"], - "null_diff": column["null_diff"], - "rel_tol": column["rel_tol"], - "abs_tol": column["abs_tol"], - } - ) - if column["unequal_cnt"] > 0: - match_sample.append( - self.sample_mismatch( - column["column"], sample_count, for_display=True - ) - ) - - if any_mismatch: - return { - "mismatch_stats": { - "has_mismatches": True, - "stats": match_stats, - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "samples": [df_to_str(sample) for sample in match_sample], - "has_samples": len(match_sample) > 0 and sample_count > 0, - } - } - return { - "mismatch_stats": { - "has_mismatches": False, - "has_samples": False, - } - } - - def _get_unique_rows_data(self, sample_count: int, column_count: int) -> dict: - """Generate data for unique rows in both dataframes. - - Parameters - ---------- - sample_count : int - Number of samples to include. - column_count : int - Number of columns to include. - - Returns - ------- - dict - Dictionary containing unique rows data for both dataframes. - """ - df1_unq_count = self.df1_unq_rows.count() - df2_unq_count = self.df2_unq_rows.count() - - min_sample_count_df1 = min(sample_count, df1_unq_count) - min_sample_count_df2 = min(sample_count, df2_unq_count) - min_column_count_df1 = min(column_count, len(self.df1_unq_rows.columns)) - min_column_count_df2 = min(column_count, len(self.df2_unq_rows.columns)) - - return { - "sample_count": sample_count, - "column_count": column_count, - "df1_unique_rows": { - "has_rows": min_sample_count_df1 > 0, - "rows": ( - df_to_str( - self.df1_unq_rows.select( - self.df1_unq_rows.columns[:min_column_count_df1] - ), - sample_count=min_sample_count_df1, - ) - if df1_unq_count > 0 - else "" - ), - "columns": ( - self.df1_unq_rows.columns[:min_column_count_df1] - if df1_unq_count > 0 - else "" - ), - }, - "df2_unique_rows": { - "has_rows": min_sample_count_df2 > 0, - "rows": ( - df_to_str( - self.df2_unq_rows.select( - self.df2_unq_rows.columns[:min_column_count_df2] - ), - sample_count=min_sample_count_df2, - ) - if df2_unq_count > 0 - else "" - ), - "columns": ( - self.df2_unq_rows.columns[:min_column_count_df2] - if df2_unq_count > 0 - else "" - ), - }, - } - - def report( - self, - sample_count: int = 10, - column_count: int = 10, - html_file: str | None = None, - template_path: str | None = None, - ) -> str: - """Return a string representation of a report. - - The representation can then be printed or saved to a file. You can customize the - report's appearance by providing a custom Jinja2 template. - - Parameters - ---------- - sample_count : int, optional - The number of sample records to return. Defaults to 10. - - column_count : int, optional - The number of columns to display in the sample records output. Defaults to 10. - - html_file : str, optional - HTML file name to save report output to. If ``None`` the file creation will be skipped. - - template_path : str, optional - Path to a custom Jinja2 template file to use for report generation. - If ``None``, the default template will be used. The template receives the - following context variables: - - - ``column_summary``: Dict with column statistics including: ``common_columns``, ``df1_unique``, ``df2_unique``, ``df1_name``, ``df2_name`` - - ``row_summary``: Dict with row statistics including: ``match_columns``, ``equal_rows``, ``unequal_rows`` - - ``column_comparison``: Dict with column comparison statistics including: ``unequal_columns``, ``equal_columns``, ``unequal_values`` - - ``mismatch_stats``: Dict containing: - - ``stats``: List of dicts with column mismatch statistics (column, match, mismatch, null_diff, etc.) - - ``samples``: Sample rows with mismatched values - - ``has_samples``: Boolean indicating if there are any mismatch samples - - ``has_mismatches``: Boolean indicating if there are any mismatches - - ``df1_unique_rows``: Dict with unique rows in df1 including: ``has_rows``, ``rows``, ``columns`` - - ``df2_unique_rows``: Dict with unique rows in df2 including: ``has_rows``, ``rows``, ``columns`` - - Returns - ------- - str - The report, formatted according to the template. - """ - # Get counts for the dataframes - df1_count = self.df1.count() - df2_count = self.df2.count() - - # Prepare template data - template_data: Dict[str, Any] = { - **self._get_column_summary(), - **self._get_row_summary(), - **self._get_column_comparison(), - **self._get_mismatch_stats(sample_count), - **self._get_unique_rows_data(sample_count, column_count), - "df1_name": self.df1_name, - "df2_name": self.df2_name, - "df1_shape": (df1_count, len(self.df1.columns)), - "df2_shape": (df2_count, len(self.df2.columns)), - "column_count": column_count, - } - - # Determine which template to use - template_name = template_path if template_path else "report_template.j2" - - # Render the main report - report = render(template_name, **template_data) - - if html_file: - save_html_report(report, html_file) - - return report + def _column_names(self, df: Any) -> List[str]: + return list(df.columns) def columns_equal( diff --git a/datacompy/templates/report_template.j2 b/datacompy/templates/report_template.j2 index f0020981..7eb03e2c 100644 --- a/datacompy/templates/report_template.j2 +++ b/datacompy/templates/report_template.j2 @@ -14,14 +14,14 @@ Column Summary -------------- Number of columns in common: {{ column_summary.common_columns }} -Number of columns in {{ column_summary.df1_name }} but not in {{ column_summary.df2_name }}: {{ column_summary.df1_unique }} -Number of columns in {{ column_summary.df2_name }} but not in {{ column_summary.df1_name }}: {{ column_summary.df2_unique }} +Number of columns in {{ column_summary.df1_name }} but not in {{ column_summary.df2_name }}: {{ column_summary.df1_unique ~ " " ~ column_summary.df1_unique_columns | list if column_summary.df1_unique_columns else column_summary.df1_unique }} +Number of columns in {{ column_summary.df2_name }} but not in {{ column_summary.df1_name }}: {{ column_summary.df2_unique ~ " " ~ column_summary.df2_unique_columns | list if column_summary.df2_unique_columns else column_summary.df2_unique }} Row Summary ----------- -Matched on: {{ row_summary.match_columns }} -Any duplicates on match values: {{ row_summary.has_duplicates }} +Matched on: {{ "index" if row_summary.on_index else row_summary.match_columns | join(", ") }} +Any duplicates on match values: {{ "Yes" if row_summary.has_duplicates else "No" }} Default Absolute Tolerance: {{ row_summary.abs_tol }} Default Relative Tolerance: {{ row_summary.rel_tol }} Number of rows in common: {{ "{:,}".format(row_summary.common_rows) }} diff --git a/docs/source/index.rst b/docs/source/index.rst index cfabc827..46ee380a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,6 +15,7 @@ Contents Snowflake Usage Polars Usage Template Guide + Report API Benchmarks Developer Instructions diff --git a/docs/source/pandas_usage.rst b/docs/source/pandas_usage.rst index 85481b7a..6d826359 100644 --- a/docs/source/pandas_usage.rst +++ b/docs/source/pandas_usage.rst @@ -39,7 +39,7 @@ Set up like: from io import StringIO import pandas as pd - from datacompy.core import Compare + from datacompy.pandas import PandasCompare data1 = """acct_id,dollar_amt,name,float_fld,date_fld 10000001234,123.45,George Maharis,14530.1555,2017-01-01 @@ -69,7 +69,7 @@ join column(s) or by index. .. code-block:: python - compare = Compare( + compare = PandasCompare( df1, df2, join_columns='acct_id', #You can also specify a list of columns @@ -80,16 +80,16 @@ join column(s) or by index. # OR - compare = Compare(df1, df2, join_columns=['acct_id', 'name']) + compare = PandasCompare(df1, df2, join_columns=['acct_id', 'name']) # OR - compare = Compare(df1, df2, on_index=True) + compare = PandasCompare(df1, df2, on_index=True) Reports ------- -A report is generated by calling ``Compare.report()``, which returns a string. +A report is generated by calling ``PandasCompare.report()``, which returns a string. Here is a sample report generated by ``datacompy`` for the two tables above, joined on ``acct_id`` (Note: if you don't specify ``df1_name`` and/or ``df2_name``, then any instance of "original" or "new" in the report is replaced with "df1" diff --git a/docs/source/report_api.rst b/docs/source/report_api.rst new file mode 100644 index 00000000..8d66f203 --- /dev/null +++ b/docs/source/report_api.rst @@ -0,0 +1,79 @@ +Report API +========== + +DataComPy's ``datacompy.report`` module provides a typed data model where +every backend produces a :class:`~datacompy.report.ReportData` instance via +``compare.build_report_data()``. Rendering methods (``render()``, +``to_html()``, ``save()``, ``to_dict()``) live directly on ``ReportData``, +making it easy to build custom dashboards, export to JSON, or plug in +alternative templates. + +Programmatic Access +------------------- + +Call ``compare.build_report_data()`` on any compare object to get a +:class:`~datacompy.report.ReportData` instance **without** triggering +any rendering: + +.. code-block:: python + + import pandas as pd + from datacompy import PandasCompare + + df1 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]}) + df2 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 99, 30]}) + + compare = PandasCompare(df1, df2, join_columns="id") + data = compare.build_report_data() + + # Inspect structured fields directly + print(data.row_summary.unequal_rows) # 1 + print(data.mismatch_stats.stats[0].column) # 'val' + print(data.column_summary.common_columns) # 2 + +The same method is available on all backends (``PolarsCompare``, +``SparkSQLCompare``, ``SnowflakeCompare``). + +Rendering and Export +-------------------- + +Rendering methods live on :class:`~datacompy.report.ReportData` directly: + +.. code-block:: python + + data = compare.build_report_data() + + # Plain-text report (same output as compare.report()) + print(data.render()) + + # Save as HTML file + data.save("comparison.html") + + # JSON-serializable dict (for dashboards / APIs) + import json + payload = json.dumps(data.to_dict()) + +Custom Templates +~~~~~~~~~~~~~~~~ + +Pass a ``template_path`` to ``render()`` or ``save()`` to use your own Jinja2 +template: + +.. code-block:: python + + print(data.render(template_path="my_report.j2")) + data.save("report.html", template_path="my_report.j2") + +The template receives the same context as the default +``report_template.j2``; see :doc:`template_guide` for the full variable +reference. + +Data Classes +------------ + +.. automodule:: datacompy.report + :members: ReportData, ColumnSummary, RowSummary, ColumnComparison, + MismatchStat, MismatchStats, UniqueRowsData + :show-inheritance: + :undoc-members: + :no-index: diff --git a/docs/source/template_guide.rst b/docs/source/template_guide.rst index bf3b3957..44388685 100644 --- a/docs/source/template_guide.rst +++ b/docs/source/template_guide.rst @@ -8,73 +8,99 @@ Template Basics --------------- Custom templates are Jinja2 templates that receive comparison data and format it according to your needs. -The template receives a context dictionary with various comparison metrics and data samples. +The template context is produced by calling ``dataclasses.asdict()`` on the +:class:`~datacompy.report.ReportData` instance, so every field is passed in +with its **typed value** — no pre-formatting is applied. All formatting +decisions belong in the template. Available Template Variables ---------------------------- -The following variables are available in the template context: +The following variables are available in the template context. They mirror +the fields of :class:`~datacompy.report.ReportData` and its nested dataclasses. +------------------------+--------------------------------------------------------------------------------+ | Variable | Description | +========================+================================================================================+ + | ``df1_name``, | Names of the dataframes being compared (str). | + | ``df2_name`` | | + +------------------------+--------------------------------------------------------------------------------+ + | ``df1_shape``, | Tuples of ``(rows, columns)``. Index ``[0]`` is row count, | + | ``df2_shape`` | ``[1]`` is column count. | + +------------------------+--------------------------------------------------------------------------------+ + | ``column_count`` | Maximum number of columns shown in unique-row sample tables (int). | + +------------------------+--------------------------------------------------------------------------------+ | ``column_summary`` | Dict with column statistics including: | | | | - | | - ``common_columns``: List of columns common to both dataframes | - | | - ``df1_unique``: List of columns unique to df1 | - | | - ``df2_unique``: List of columns unique to df2 | - | | - ``df1_name``: Name of the first dataframe | - | | - ``df2_name``: Name of the second dataframe | + | | - ``common_columns`` (int): count of columns in both DataFrames | + | | - ``df1_unique`` (int): number of columns only in df1 | + | | - ``df1_unique_columns`` (list of str): names of those columns | + | | - ``df2_unique`` (int): number of columns only in df2 | + | | - ``df2_unique_columns`` (list of str): names of those columns | + | | - ``df1_name``, ``df2_name`` (str): DataFrame labels | +------------------------+--------------------------------------------------------------------------------+ | ``row_summary`` | Dict with row statistics including: | | | | - | | - ``match_columns``: List of columns used for matching | - | | - ``abs_tol``: Absolute tolerance between two values | - | | - ``rel_tol``: Relative tolerance between two values | - | | - ``common_rows``: Number of rows in common | - | | - ``df1_unique``: Number of rows unique to df1 | - | | - ``df2_unique``: Number of rows unique to df2 | - | | - ``unequal_rows``: Number of rows with differences | - | | - ``df1_name``: Name of the first dataframe | - | | - ``df2_name``: Name of the second dataframe | + | | - ``match_columns`` (list of str): join column names; empty | + | | list when matching on index | + | | - ``on_index`` (bool): true when the comparison was joined on index | + | | - ``has_duplicates`` (bool): true when duplicate join-key values exist | + | | - ``abs_tol``, ``rel_tol``: float (global) or | + | | ``dict[str, float]`` per-column tolerances | + | | - ``common_rows`` (int): rows present in both DataFrames | + | | - ``df1_unique``, ``df2_unique`` (int): row counts unique to | + | | each side | + | | - ``unequal_rows`` (int): common rows where at least one | + | | compared column differs | + | | - ``equal_rows`` (int): common rows where all compared columns | + | | match | + | | - ``df1_name``, ``df2_name`` (str): DataFrame labels | +------------------------+--------------------------------------------------------------------------------+ | ``column_comparison`` | Dict with column comparison stats: | | | | - | | - ``unequal_columns``: List of columns with mismatches | - | | - ``equal_columns``: List of columns that match exactly | - | | - ``unequal_values``: Count of unequal values across all columns | + | | - ``unequal_columns`` (int): columns with at least one | + | | unequal value | + | | - ``equal_columns`` (int): columns where all values match | + | | - ``unequal_values`` (int): total individual cell mismatches | +------------------------+--------------------------------------------------------------------------------+ | ``mismatch_stats`` | Dict containing: | | | | - | | - ``stats``: List of dicts with per-column mismatch statistics: | - | | - ``column``: Column name | - | | - ``match``: Number of matching values | - | | - ``mismatch``: Number of mismatched values | - | | - ``null_diff``: Number of null value differences | - | | - ``total``: Total number of comparisons | - | | - ``samples``: Sample rows with mismatched values | - | | - ``has_samples``: Boolean indicating if there are any samples | - | | - ``has_mismatches``: Boolean indicating if there are any mismatches | - +------------------------+--------------------------------------------------------------------------------+ - | ``df1_unique_rows`` | Dict with unique rows in df1: | - | | | - | | - ``has_rows``: Boolean indicating if there are unique rows | - | | - ``rows``: Sample of unique rows (as strings) | - | | - ``columns``: List of column names | - +------------------------+--------------------------------------------------------------------------------+ - | ``df2_unique_rows`` | Dict with unique rows in df2: | - | | | - | | - ``has_rows``: Boolean indicating if there are unique rows | - | | - ``rows``: Sample of unique rows (as strings) | - | | - ``columns``: List of column names | + | | - ``has_mismatches`` (bool): true when at least one column | + | | has unequal values or types | + | | - ``has_samples`` (bool): true when sample rows are available | + | | - ``stats`` (list of dict): one entry per mismatched column, | + | | sorted by column name. Each dict has ``column`` (str), | + | | ``dtype1`` (str), ``dtype2`` (str), ``unequal_cnt`` (int), | + | | ``max_diff`` (float), ``null_diff`` (int), | + | | ``rel_tol`` (float), ``abs_tol`` (float) | + | | - ``samples`` (list of str): pre-rendered ASCII tables of | + | | sample mismatched rows, one per column with mismatches | + | | - ``df1_name``, ``df2_name`` (str): DataFrame labels | +------------------------+--------------------------------------------------------------------------------+ - | ``df1_shape``, | Tuples of (rows, columns) for each dataframe | - | ``df2_shape`` | | - +------------------------+--------------------------------------------------------------------------------+ - | ``df1_name``, | Names of the dataframes being compared | - | ``df2_name`` | | + | ``df1_unique_rows``, | Dict with sample rows present in only one DataFrame: | + | ``df2_unique_rows`` | | + | | - ``has_rows`` (bool): true when at least one unique row exists | + | | - ``rows`` (str): pre-rendered ASCII table; empty string when | + | | there are no unique rows | +------------------------+--------------------------------------------------------------------------------+ +Formatting typed values in templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because field values are typed (not pre-formatted), use Jinja2 expressions +to produce display strings where needed. Three common patterns: + +.. code-block:: jinja + + {# match_columns is a list; on_index is a bool #} + Matched on: {{ "index" if row_summary.on_index else row_summary.match_columns | join(", ") }} + + {# has_duplicates is a bool #} + Duplicates: {{ "Yes" if row_summary.has_duplicates else "No" }} + + {# df1_unique is an int; df1_unique_columns is a list #} + Unique to df1: {{ column_summary.df1_unique ~ " " ~ column_summary.df1_unique_columns if column_summary.df1_unique_columns else column_summary.df1_unique }} + Creating a Custom Template -------------------------- @@ -93,26 +119,27 @@ Here's a simple example template that shows basic comparison metrics: .. code-block:: jinja # Data Comparison Report - ===================== + ======================= ## DataFrames - - {{ df1_name }}: {{ df1_shape[0] }} rows × {{ df1_shape[1] }} columns - - {{ df2_name }}: {{ df2_shape[0] }} rows × {{ df2_shape[1] }} columns + - {{ df1_name }}: {{ df1_shape[0] }} rows x {{ df1_shape[1] }} columns + - {{ df2_name }}: {{ df2_shape[0] }} rows x {{ df2_shape[1] }} columns ## Column Summary - Common columns: {{ column_summary.common_columns }} - - Columns only in {{ df1_name }}: {{ column_summary.df1_unique }} - - Columns only in {{ df2_name }}: {{ column_summary.df2_unique }} + - Columns only in {{ df1_name }}: {{ column_summary.df1_unique ~ " " ~ column_summary.df1_unique_columns if column_summary.df1_unique_columns else column_summary.df1_unique }} + - Columns only in {{ df2_name }}: {{ column_summary.df2_unique ~ " " ~ column_summary.df2_unique_columns if column_summary.df2_unique_columns else column_summary.df2_unique }} ## Row Summary - - Rows with some columns unequal: {{ row_summary.unequal_rows }} + - Matched on: {{ "index" if row_summary.on_index else row_summary.match_columns | join(", ") }} + - Rows in common: {{ row_summary.common_rows }} - Rows with all columns equal: {{ row_summary.equal_rows }} + - Rows with some columns unequal: {{ row_summary.unequal_rows }} - {% if mismatch_stats %} + {% if mismatch_stats.has_mismatches %} ## Mismatched Columns - {% for col in mismatch_stats %} - - {{ col.column }}: {{ col.unequal_cnt }} mismatches - - Match rate: {{ "%.2f"|format(col.match_rate * 100) }}% + {% for col in mismatch_stats.stats %} + - {{ col.column }}: {{ col.unequal_cnt }} unequal value(s), max diff {{ "%.4f"|format(col.max_diff) }} {% endfor %} {% endif %} @@ -123,9 +150,9 @@ To use your custom template, pass its path to the ``report()`` method: .. code-block:: python - from datacompy.core import Compare + from datacompy import PandasCompare - compare = Compare(df1, df2, join_columns=['id']) + compare = PandasCompare(df1, df2, join_columns=['id']) # Generate report with custom template report = compare.report(template_path='path/to/your/template.j2') diff --git a/tests/snapshots/pandas_duplicates.txt b/tests/snapshots/pandas_duplicates.txt new file mode 100644 index 00000000..8a5d1168 --- /dev/null +++ b/tests/snapshots/pandas_duplicates.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 3 +df2 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: Yes +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 3 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/pandas_no_mismatches.txt b/tests/snapshots/pandas_no_mismatches.txt new file mode 100644 index 00000000..0269ec5a --- /dev/null +++ b/tests/snapshots/pandas_no_mismatches.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +left 3 3 +right 3 3 + + +Column Summary +-------------- + +Number of columns in common: 3 +Number of columns in left but not in right: 0 +Number of columns in right but not in left: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in left but not in right: 0 +Number of rows in right but not in left: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 3 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 3 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/pandas_on_index.txt b/tests/snapshots/pandas_on_index.txt new file mode 100644 index 00000000..c3e596d4 --- /dev/null +++ b/tests/snapshots/pandas_on_index.txt @@ -0,0 +1,53 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 1 3 +df2 1 3 + + +Column Summary +-------------- + +Number of columns in common: 1 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: index +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 1 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 1 +Number of columns compared with all values equal: 0 +Total number of values which compare unequal: 1 + +Columns with Unequal Values or Types +------------------------------------ + +Column df1 dtype df2 dtype # Unequal Max Diff # Null Diff Rel Tol Abs Tol +-------------------- --------------- --------------- ---------- ---------- ------------ ---------- ---------- +val int64 int64 1 79.0000 0 0.0000 0.0000 + + +Sample Rows with Unequal Values +------------------------------- + + val (df1) val (df2) +0 20 99 \ No newline at end of file diff --git a/tests/snapshots/pandas_sample_count.txt b/tests/snapshots/pandas_sample_count.txt new file mode 100644 index 00000000..991dd359 --- /dev/null +++ b/tests/snapshots/pandas_sample_count.txt @@ -0,0 +1,55 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 3 +df2 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 3 +Number of rows with all compared columns equal: 0 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 1 +Number of columns compared with all values equal: 1 +Total number of values which compare unequal: 3 + +Columns with Unequal Values or Types +------------------------------------ + +Column df1 dtype df2 dtype # Unequal Max Diff # Null Diff Rel Tol Abs Tol +-------------------- --------------- --------------- ---------- ---------- ------------ ---------- ---------- +val int64 int64 3 3.0000 0 0.0000 0.0000 + + +Sample Rows with Unequal Values +------------------------------- + + id val (df1) val (df2) +0 1 10 11 +1 2 20 22 +2 3 30 33 diff --git a/tests/snapshots/pandas_sample_count_zero.txt b/tests/snapshots/pandas_sample_count_zero.txt new file mode 100644 index 00000000..1fa23ab8 --- /dev/null +++ b/tests/snapshots/pandas_sample_count_zero.txt @@ -0,0 +1,46 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 3 +df2 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 3 +Number of rows with all compared columns equal: 0 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 1 +Number of columns compared with all values equal: 1 +Total number of values which compare unequal: 3 + +Columns with Unequal Values or Types +------------------------------------ + +Column df1 dtype df2 dtype # Unequal Max Diff # Null Diff Rel Tol Abs Tol +-------------------- --------------- --------------- ---------- ---------- ------------ ---------- ---------- +val int64 int64 3 3.0000 0 0.0000 0.0000 \ No newline at end of file diff --git a/tests/snapshots/pandas_unique_columns.txt b/tests/snapshots/pandas_unique_columns.txt new file mode 100644 index 00000000..6f69fc13 --- /dev/null +++ b/tests/snapshots/pandas_unique_columns.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 3 2 +df2 3 2 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 1 ['only_1'] +Number of columns in df2 but not in df1: 1 ['only_2'] + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 2 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/pandas_unique_rows.txt b/tests/snapshots/pandas_unique_rows.txt new file mode 100644 index 00000000..a489b095 --- /dev/null +++ b/tests/snapshots/pandas_unique_rows.txt @@ -0,0 +1,52 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 3 +df2 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 2 +Number of rows in df1 but not in df2: 1 +Number of rows in df2 but not in df1: 1 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 + + +Sample Rows Only in df1 (First 10 Columns) +------------------------------------------ + + id val +0 3 30.0 + +Sample Rows Only in df2 (First 10 Columns) +------------------------------------------ + + id val +0 4 40.0 \ No newline at end of file diff --git a/tests/snapshots/pandas_with_mismatches.txt b/tests/snapshots/pandas_with_mismatches.txt new file mode 100644 index 00000000..9e4209d8 --- /dev/null +++ b/tests/snapshots/pandas_with_mismatches.txt @@ -0,0 +1,57 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 3 3 +df2 3 3 + + +Column Summary +-------------- + +Number of columns in common: 3 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 1 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 2 +Number of columns compared with all values equal: 1 +Total number of values which compare unequal: 2 + +Columns with Unequal Values or Types +------------------------------------ + +Column df1 dtype df2 dtype # Unequal Max Diff # Null Diff Rel Tol Abs Tol +-------------------- --------------- --------------- ---------- ---------- ------------ ---------- ---------- +score float64 float64 1 0.5000 0 0.0000 0.0000 +val int64 int64 1 79.0000 0 0.0000 0.0000 + + +Sample Rows with Unequal Values +------------------------------- + + id val (df1) val (df2) +0 2 20 99 + + id score (df1) score (df2) +0 2 2.0 2.5 \ No newline at end of file diff --git a/tests/snapshots/pandas_with_tolerances.txt b/tests/snapshots/pandas_with_tolerances.txt new file mode 100644 index 00000000..1fecc505 --- /dev/null +++ b/tests/snapshots/pandas_with_tolerances.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 2 +df2 2 2 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0.001 +Default Relative Tolerance: 0 +Number of rows in common: 2 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/polars_no_mismatches.txt b/tests/snapshots/polars_no_mismatches.txt new file mode 100644 index 00000000..4dd7514b --- /dev/null +++ b/tests/snapshots/polars_no_mismatches.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +left 2 3 +right 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in left but not in right: 0 +Number of columns in right but not in left: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in left but not in right: 0 +Number of rows in right but not in left: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 3 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/polars_unique_columns.txt b/tests/snapshots/polars_unique_columns.txt new file mode 100644 index 00000000..6f69fc13 --- /dev/null +++ b/tests/snapshots/polars_unique_columns.txt @@ -0,0 +1,39 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 3 2 +df2 3 2 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 1 ['only_1'] +Number of columns in df2 but not in df1: 1 ['only_2'] + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 2 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 \ No newline at end of file diff --git a/tests/snapshots/polars_unique_rows.txt b/tests/snapshots/polars_unique_rows.txt new file mode 100644 index 00000000..a516f0d5 --- /dev/null +++ b/tests/snapshots/polars_unique_rows.txt @@ -0,0 +1,64 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 2 3 +df2 2 3 + + +Column Summary +-------------- + +Number of columns in common: 2 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 2 +Number of rows in df1 but not in df2: 1 +Number of rows in df2 but not in df1: 1 + +Number of rows with some compared columns unequal: 0 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 0 +Number of columns compared with all values equal: 2 +Total number of values which compare unequal: 0 + + +Sample Rows Only in df1 (First 10 Columns) +------------------------------------------ + +shape: (1, 2) +┌─────┬─────┐ +│ id ┆ val │ +│ --- ┆ --- │ +│ i64 ┆ i64 │ +╞═════╪═════╡ +│ 3 ┆ 30 │ +└─────┴─────┘ + +Sample Rows Only in df2 (First 10 Columns) +------------------------------------------ + +shape: (1, 2) +┌─────┬─────┐ +│ id ┆ val │ +│ --- ┆ --- │ +│ i64 ┆ i64 │ +╞═════╪═════╡ +│ 4 ┆ 40 │ +└─────┴─────┘ \ No newline at end of file diff --git a/tests/snapshots/polars_with_mismatches.txt b/tests/snapshots/polars_with_mismatches.txt new file mode 100644 index 00000000..52f89939 --- /dev/null +++ b/tests/snapshots/polars_with_mismatches.txt @@ -0,0 +1,69 @@ +DataComPy Comparison +------------------- + +DataFrame Summary +----------------- + +DataFrame Columns Rows +------------------------ +df1 3 3 +df2 3 3 + + +Column Summary +-------------- + +Number of columns in common: 3 +Number of columns in df1 but not in df2: 0 +Number of columns in df2 but not in df1: 0 + +Row Summary +----------- + +Matched on: id +Any duplicates on match values: No +Default Absolute Tolerance: 0 +Default Relative Tolerance: 0 +Number of rows in common: 3 +Number of rows in df1 but not in df2: 0 +Number of rows in df2 but not in df1: 0 + +Number of rows with some compared columns unequal: 1 +Number of rows with all compared columns equal: 2 + +Column Comparison +----------------- + +Number of columns compared with some values unequal: 2 +Number of columns compared with all values equal: 1 +Total number of values which compare unequal: 2 + +Columns with Unequal Values or Types +------------------------------------ + +Column df1 dtype df2 dtype # Unequal Max Diff # Null Diff Rel Tol Abs Tol +-------------------- --------------- --------------- ---------- ---------- ------------ ---------- ---------- +score Float64 Float64 1 0.5000 0 0.0000 0.0000 +val Int64 Int64 1 79.0000 0 0.0000 0.0000 + + +Sample Rows with Unequal Values +------------------------------- + +shape: (1, 3) +┌─────┬───────────┬───────────┐ +│ id ┆ val (df1) ┆ val (df2) │ +│ --- ┆ --- ┆ --- │ +│ i64 ┆ i64 ┆ i64 │ +╞═════╪═══════════╪═══════════╡ +│ 2 ┆ 20 ┆ 99 │ +└─────┴───────────┴───────────┘ + +shape: (1, 3) +┌─────┬─────────────┬─────────────┐ +│ id ┆ score (df1) ┆ score (df2) │ +│ --- ┆ --- ┆ --- │ +│ i64 ┆ f64 ┆ f64 │ +╞═════╪═════════════╪═════════════╡ +│ 2 ┆ 2.0 ┆ 2.5 │ +└─────┴─────────────┴─────────────┘ \ No newline at end of file diff --git a/tests/test_pandas.py b/tests/test_pandas.py index 69fc4e2c..ba1a4877 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -1378,8 +1378,8 @@ def test_integer_column_names(): assert compare.matches() -@mock.patch("datacompy.pandas.render") -@mock.patch("datacompy.pandas.save_html_report") +@mock.patch("datacompy.report.render") +@mock.patch("datacompy.base.save_html_report") def test_save_html(mock_save_html, mock_render): df1 = pd.DataFrame([{"a": 1, "b": 2}, {"a": 2, "b": 3}]) df2 = pd.DataFrame([{"a": 1, "b": 2}, {"a": 2, "b": 4}]) diff --git a/tests/test_polars.py b/tests/test_polars.py index 13bb67e5..54591f4a 100644 --- a/tests/test_polars.py +++ b/tests/test_polars.py @@ -1322,8 +1322,8 @@ def test_lower(): ) -@mock.patch("datacompy.polars.render") -@mock.patch("datacompy.polars.save_html_report") +@mock.patch("datacompy.report.render") +@mock.patch("datacompy.base.save_html_report") def test_save_html(mock_save_html, mock_render): df1 = pl.DataFrame([{"a": 1, "b": 2}, {"a": 1, "b": 2}]) df2 = pl.DataFrame([{"a": 1, "b": 2}, {"a": 1, "b": 2}]) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 00000000..342bfd1e --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,345 @@ +"""Unit tests for datacompy.report dataclasses and Report class.""" + +import dataclasses +import json +import os + +import pandas as pd +import polars as pl +import pytest +from datacompy import ( + ColumnComparison, + ColumnSummary, + MismatchStat, + MismatchStats, + PandasCompare, + PolarsCompare, + ReportData, + RowSummary, + UniqueRowsData, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _minimal_report_data( + df1_name: str = "df1", + df2_name: str = "df2", + unequal_rows: int = 0, + mismatch: bool = False, +) -> ReportData: + stat = ( + ( + MismatchStat( + column="val", + dtype1="int64", + dtype2="int64", + unequal_cnt=1, + max_diff=1.0, + null_diff=0, + rel_tol=0.0, + abs_tol=0.0, + ), + ) + if mismatch + else () + ) + + return ReportData( + df1_name=df1_name, + df2_name=df2_name, + df1_shape=(3, 2), + df2_shape=(3, 2), + column_count=10, + column_summary=ColumnSummary( + common_columns=2, + df1_unique=0, + df1_unique_columns=(), + df2_unique=0, + df2_unique_columns=(), + df1_name=df1_name, + df2_name=df2_name, + ), + row_summary=RowSummary( + match_columns=("id",), + on_index=False, + has_duplicates=False, + abs_tol=0, + rel_tol=0, + common_rows=3, + df1_unique=0, + df2_unique=0, + unequal_rows=unequal_rows, + equal_rows=3 - unequal_rows, + df1_name=df1_name, + df2_name=df2_name, + ), + column_comparison=ColumnComparison( + unequal_columns=1 if mismatch else 0, + equal_columns=1 if not mismatch else 0, + unequal_values=1 if mismatch else 0, + ), + mismatch_stats=MismatchStats( + has_mismatches=mismatch, + has_samples=mismatch, + stats=stat, + samples=("col df1 df2\n 0 1 2",) if mismatch else (), + df1_name=df1_name, + df2_name=df2_name, + ), + df1_unique_rows=UniqueRowsData(has_rows=False), + df2_unique_rows=UniqueRowsData(has_rows=False), + ) + + +# --------------------------------------------------------------------------- +# Dataclass tests +# --------------------------------------------------------------------------- + + +def test_report_data_is_frozen(): + data = _minimal_report_data() + with pytest.raises((dataclasses.FrozenInstanceError, AttributeError)): + data.df1_name = "changed" # type: ignore[misc] + + +def test_mismatch_stat_frozen(): + stat = MismatchStat("col", "int64", "int64", 1, 1.0, 0, 0.0, 0.0) + with pytest.raises((dataclasses.FrozenInstanceError, AttributeError)): + stat.column = "other" # type: ignore[misc] + + +def test_mismatch_stats_defaults(): + ms = MismatchStats(has_mismatches=False, has_samples=False) + assert ms.stats == () + assert ms.samples == () + assert ms.df1_name == "" + + +def test_unique_rows_data_defaults(): + u = UniqueRowsData(has_rows=False) + assert u.rows == "" + + +# --------------------------------------------------------------------------- +# Template rendering formatting tests +# (formatting previously done in to_template_context, now done in the template) +# --------------------------------------------------------------------------- + + +def test_render_match_columns_joined(): + assert "Matched on: id" in _minimal_report_data().render() + + +def test_render_match_columns_on_index(): + data = dataclasses.replace( + _minimal_report_data(), + row_summary=dataclasses.replace( + _minimal_report_data().row_summary, on_index=True + ), + ) + assert "Matched on: index" in data.render() + + +def test_render_match_columns_multi_column(): + data = dataclasses.replace( + _minimal_report_data(), + row_summary=dataclasses.replace( + _minimal_report_data().row_summary, + match_columns=("id", "name"), + on_index=False, + ), + ) + assert "Matched on: id, name" in data.render() + + +def test_render_has_duplicates_yes(): + data = dataclasses.replace( + _minimal_report_data(), + row_summary=dataclasses.replace( + _minimal_report_data().row_summary, has_duplicates=True + ), + ) + assert "Any duplicates on match values: Yes" in data.render() + + +def test_render_has_duplicates_no(): + assert "Any duplicates on match values: No" in _minimal_report_data().render() + + +def test_render_unique_columns_formatted_with_names(): + data = dataclasses.replace( + _minimal_report_data(), + column_summary=dataclasses.replace( + _minimal_report_data().column_summary, + df1_unique=2, + df1_unique_columns=("col_a", "col_b"), + ), + ) + assert "2 ['col_a', 'col_b']" in data.render() + + +def test_render_unique_columns_plain_zero_when_none(): + assert "but not in df2: 0\n" in _minimal_report_data().render() + + +def test_render_mismatch_stats_column_name_present(): + assert "val" in _minimal_report_data(mismatch=True).render() + + +# --------------------------------------------------------------------------- +# ReportData rendering / export tests +# --------------------------------------------------------------------------- + + +def test_render_returns_string(): + text = _minimal_report_data().render() + assert isinstance(text, str) + assert "DataComPy" in text + + +def test_str_equals_render(): + data = _minimal_report_data() + assert str(data) == data.render() + + +def test_repr(): + r = repr(_minimal_report_data("left", "right")) + assert "left" in r + assert "right" in r + + +def test_to_html_contains_pre(): + html = _minimal_report_data().to_html() + assert "
" in html
+    assert "" in html.lower()
+
+
+def test_save_writes_html_file(tmp_path):
+    dest = tmp_path / "report.html"
+    _minimal_report_data().save(dest)
+    assert dest.exists()
+    assert "
" in dest.read_text()
+
+
+def test_to_dict_json_serializable():
+    d = _minimal_report_data(mismatch=True).to_dict()
+    assert "df1_name" in json.dumps(d)
+
+
+def test_to_dict_roundtrip_fields():
+    d = _minimal_report_data(mismatch=True).to_dict()
+    assert d["row_summary"]["common_rows"] == 3
+    assert d["mismatch_stats"]["stats"][0]["column"] == "val"
+
+
+def test_render_with_custom_template(tmp_path):
+    tmpl = tmp_path / "custom.j2"
+    tmpl.write_text("{{ df1_name }} vs {{ df2_name }}")
+    assert (
+        _minimal_report_data("left", "right").render(template_path=str(tmpl))
+        == "left vs right"
+    )
+
+
+def test_render_with_mismatches():
+    text = _minimal_report_data(mismatch=True).render()
+    assert "Columns with Unequal" in text
+    assert "val" in text
+
+
+def test_render_no_mismatches():
+    assert "Columns with Unequal" not in _minimal_report_data(mismatch=False).render()
+
+
+# ---------------------------------------------------------------------------
+# Integration: build_report_data roundtrip via pandas and polars
+# ---------------------------------------------------------------------------
+
+
+def test_pandas_returns_report_data():
+    df1 = pd.DataFrame({"id": [1, 2, 3], "val": [1, 2, 3]})
+    df2 = pd.DataFrame({"id": [1, 2, 3], "val": [1, 9, 3]})
+    data = PandasCompare(df1, df2, "id").build_report_data()
+    assert isinstance(data, ReportData)
+    assert data.df1_shape == (3, 2)
+    assert data.row_summary.common_rows == 3
+    assert data.row_summary.unequal_rows == 1
+    assert data.mismatch_stats.has_mismatches is True
+    assert data.mismatch_stats.stats[0].column == "val"
+
+
+def test_pandas_mismatch_stats_sorted():
+    df1 = pd.DataFrame({"id": [1, 2], "b": [1, 2], "a": [10, 20]})
+    df2 = pd.DataFrame({"id": [1, 2], "b": [9, 9], "a": [99, 99]})
+    data = PandasCompare(df1, df2, "id").build_report_data()
+    cols = [s.column for s in data.mismatch_stats.stats]
+    assert cols == sorted(cols)
+
+
+def test_pandas_no_mismatches():
+    df1 = pd.DataFrame({"id": [1, 2], "val": [1, 2]})
+    df2 = pd.DataFrame({"id": [1, 2], "val": [1, 2]})
+    data = PandasCompare(df1, df2, "id").build_report_data()
+    assert data.mismatch_stats.has_mismatches is False
+    assert data.mismatch_stats.stats == ()
+
+
+def test_pandas_unique_columns():
+    df1 = pd.DataFrame({"id": [1], "only_in_1": [9]})
+    df2 = pd.DataFrame({"id": [1], "only_in_2": [9]})
+    data = PandasCompare(df1, df2, "id").build_report_data()
+    assert data.column_summary.df1_unique_columns == ("only_in_1",)
+    assert data.column_summary.df2_unique_columns == ("only_in_2",)
+
+
+def test_pandas_unique_rows():
+    df1 = pd.DataFrame({"id": [1, 2, 3], "val": [1, 2, 3]})
+    df2 = pd.DataFrame({"id": [1, 2], "val": [1, 2]})
+    data = PandasCompare(df1, df2, "id").build_report_data()
+    assert data.df1_unique_rows.has_rows is True
+    assert "3" in data.df1_unique_rows.rows
+    assert data.df2_unique_rows.has_rows is False
+
+
+def test_pandas_on_index():
+    df1 = pd.DataFrame({"val": [1, 2]}, index=[0, 1])
+    df2 = pd.DataFrame({"val": [1, 9]}, index=[0, 1])
+    data = PandasCompare(df1, df2, on_index=True).build_report_data()
+    assert data.row_summary.on_index is True
+    assert "Matched on: index" in data.render()
+
+
+def test_pandas_html_file(tmp_path):
+    df1 = pd.DataFrame({"id": [1], "val": [1]})
+    df2 = pd.DataFrame({"id": [1], "val": [1]})
+    html_path = str(tmp_path / "report.html")
+    text = PandasCompare(df1, df2, "id").report(html_file=html_path)
+    assert isinstance(text, str)
+    assert os.path.exists(html_path)
+
+
+def test_polars_returns_report_data():
+    df1 = pl.DataFrame({"id": [1, 2], "val": [10, 20]})
+    df2 = pl.DataFrame({"id": [1, 2], "val": [10, 99]})
+    data = PolarsCompare(df1, df2, "id").build_report_data()
+    assert isinstance(data, ReportData)
+    assert data.mismatch_stats.has_mismatches is True
+
+
+def test_polars_mismatch_stats_sorted():
+    df1 = pl.DataFrame({"id": [1], "z_col": [1], "a_col": [10]})
+    df2 = pl.DataFrame({"id": [1], "z_col": [9], "a_col": [99]})
+    data = PolarsCompare(df1, df2, "id").build_report_data()
+    cols = [s.column for s in data.mismatch_stats.stats]
+    assert cols == sorted(cols)
+
+
+def test_report_method_returns_string():
+    df1 = pd.DataFrame({"id": [1, 2], "val": [1, 2]})
+    df2 = pd.DataFrame({"id": [1, 2], "val": [1, 9]})
+    rpt = PandasCompare(df1, df2, "id").report()
+    assert isinstance(rpt, str)
+    assert "DataComPy" in rpt
diff --git a/tests/test_report_snapshots.py b/tests/test_report_snapshots.py
new file mode 100644
index 00000000..159729d8
--- /dev/null
+++ b/tests/test_report_snapshots.py
@@ -0,0 +1,132 @@
+"""Snapshot regression tests for report() output across backends.
+
+Each test case generates a report and compares it against a committed snapshot
+in tests/snapshots/.  To regenerate all snapshots:
+
+    DATACOMPY_REGEN_SNAPSHOTS=1 pytest tests/test_report_snapshots.py
+
+Snapshots are named ``{backend}_{case}.txt``.  Any intentional change to
+report output (e.g. adding a new section) must be accompanied by a snapshot
+update committed alongside the code change.
+"""
+
+import os
+from pathlib import Path
+
+import pandas as pd
+import polars as pl
+from datacompy import PandasCompare, PolarsCompare
+
+SNAPSHOT_DIR = Path(__file__).parent / "snapshots"
+REGEN = os.environ.get("DATACOMPY_REGEN_SNAPSHOTS") == "1"
+
+
+def _assert_or_regen(name: str, text: str) -> None:
+    path = SNAPSHOT_DIR / f"{name}.txt"
+    if REGEN or not path.exists():
+        SNAPSHOT_DIR.mkdir(parents=True, exist_ok=True)
+        path.write_text(text, encoding="utf-8")
+        return
+    expected = path.read_text(encoding="utf-8")
+    assert text == expected, (
+        f"Report output for '{name}' changed.\n"
+        f"Run with DATACOMPY_REGEN_SNAPSHOTS=1 if the change is intentional.\n"
+        f"First diff at char {next((i for i, (a, b) in enumerate(zip(text, expected, strict=False)) if a != b), len(min(text, expected, key=len)))}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Pandas
+# ---------------------------------------------------------------------------
+
+
+class TestPandasSnapshots:
+    def test_no_mismatches(self):
+        df1 = pd.DataFrame(
+            {"id": [1, 2, 3], "val": [10, 20, 30], "name": ["a", "b", "c"]}
+        )
+        df2 = df1.copy()
+        c = PandasCompare(df1, df2, "id", df1_name="left", df2_name="right")
+        _assert_or_regen("pandas_no_mismatches", c.report())
+
+    def test_with_mismatches(self):
+        df1 = pd.DataFrame(
+            {"id": [1, 2, 3], "val": [10, 20, 30], "score": [1.0, 2.0, 3.0]}
+        )
+        df2 = pd.DataFrame(
+            {"id": [1, 2, 3], "val": [10, 99, 30], "score": [1.0, 2.5, 3.0]}
+        )
+        c = PandasCompare(df1, df2, "id")
+        _assert_or_regen("pandas_with_mismatches", c.report())
+
+    def test_unique_rows(self):
+        df1 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]})
+        df2 = pd.DataFrame({"id": [1, 2, 4], "val": [10, 20, 40]})
+        c = PandasCompare(df1, df2, "id")
+        _assert_or_regen("pandas_unique_rows", c.report())
+
+    def test_unique_columns(self):
+        df1 = pd.DataFrame({"id": [1, 2], "shared": [1, 2], "only_1": [9, 9]})
+        df2 = pd.DataFrame({"id": [1, 2], "shared": [1, 2], "only_2": [8, 8]})
+        c = PandasCompare(df1, df2, "id")
+        _assert_or_regen("pandas_unique_columns", c.report())
+
+    def test_on_index(self):
+        df1 = pd.DataFrame({"val": [10, 20, 30]})
+        df2 = pd.DataFrame({"val": [10, 99, 30]})
+        c = PandasCompare(df1, df2, on_index=True)
+        _assert_or_regen("pandas_on_index", c.report())
+
+    def test_with_tolerances(self):
+        df1 = pd.DataFrame({"id": [1, 2], "val": [1.0, 2.0]})
+        df2 = pd.DataFrame({"id": [1, 2], "val": [1.0001, 2.0001]})
+        c = PandasCompare(df1, df2, "id", abs_tol=0.001)
+        _assert_or_regen("pandas_with_tolerances", c.report())
+
+    def test_duplicates(self):
+        df1 = pd.DataFrame({"id": [1, 1, 2], "val": [10, 20, 30]})
+        df2 = pd.DataFrame({"id": [1, 1, 2], "val": [10, 20, 30]})
+        c = PandasCompare(df1, df2, "id")
+        _assert_or_regen("pandas_duplicates", c.report())
+
+    def test_sample_count_zero(self):
+        # sample_count=0 disables sample rows — output is fully deterministic
+        df1 = pd.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]})
+        df2 = pd.DataFrame({"id": [1, 2, 3], "val": [11, 22, 33]})
+        c = PandasCompare(df1, df2, "id")
+        _assert_or_regen("pandas_sample_count_zero", c.report(sample_count=0))
+
+
+# ---------------------------------------------------------------------------
+# Polars
+# ---------------------------------------------------------------------------
+
+
+class TestPolarsSnapshots:
+    def test_no_mismatches(self):
+        df1 = pl.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]})
+        df2 = df1.clone()
+        c = PolarsCompare(df1, df2, "id", df1_name="left", df2_name="right")
+        _assert_or_regen("polars_no_mismatches", c.report())
+
+    def test_with_mismatches(self):
+        df1 = pl.DataFrame(
+            {"id": [1, 2, 3], "val": [10, 20, 30], "score": [1.0, 2.0, 3.0]}
+        )
+        df2 = pl.DataFrame(
+            {"id": [1, 2, 3], "val": [10, 99, 30], "score": [1.0, 2.5, 3.0]}
+        )
+        c = PolarsCompare(df1, df2, "id")
+        _assert_or_regen("polars_with_mismatches", c.report())
+
+    def test_unique_rows(self):
+        df1 = pl.DataFrame({"id": [1, 2, 3], "val": [10, 20, 30]})
+        df2 = pl.DataFrame({"id": [1, 2, 4], "val": [10, 20, 40]})
+        c = PolarsCompare(df1, df2, "id")
+        _assert_or_regen("polars_unique_rows", c.report())
+
+    def test_unique_columns(self):
+        df1 = pl.DataFrame({"id": [1, 2], "shared": [1, 2], "only_1": [9, 9]})
+        df2 = pl.DataFrame({"id": [1, 2], "shared": [1, 2], "only_2": [8, 8]})
+        c = PolarsCompare(df1, df2, "id")
+        _assert_or_regen("polars_unique_columns", c.report())
diff --git a/tests/test_snowflake.py b/tests/test_snowflake.py
index b7d6dfa4..b7cc26a7 100644
--- a/tests/test_snowflake.py
+++ b/tests/test_snowflake.py
@@ -1590,8 +1590,8 @@ def test_generate_id_within_group_single_join(snowflake_session):
     assert (actual["_TEMP_0"] == expected).all()
 
 
-@mock.patch("datacompy.snowflake.render")
-@mock.patch("datacompy.snowflake.save_html_report")
+@mock.patch("datacompy.report.render")
+@mock.patch("datacompy.base.save_html_report")
 def test_save_html(mock_save_html, mock_render, snowflake_session):
     df1 = snowflake_session.createDataFrame([{"A": 1, "B": 2}, {"A": 1, "B": 2}])
     df2 = snowflake_session.createDataFrame([{"A": 1, "B": 2}, {"A": 1, "B": 2}])
@@ -1878,8 +1878,8 @@ def test_template_context_variables(snowflake_session):
             os.unlink(template_path)
 
 
-@mock.patch("datacompy.snowflake.save_html_report")
-@mock.patch("datacompy.snowflake.render")
+@mock.patch("datacompy.base.save_html_report")
+@mock.patch("datacompy.report.render")
 def test_html_report_generation(mock_render, mock_save_html, snowflake_session):
     """Test that HTML reports can be generated and saved to a file."""
     df1 = snowflake_session.createDataFrame([("a", 1), ("b", 2)], ["id", "value"])
diff --git a/tests/test_spark.py b/tests/test_spark.py
index 2723b7d0..eb620361 100644
--- a/tests/test_spark.py
+++ b/tests/test_spark.py
@@ -1568,8 +1568,8 @@ def test_integer_column_names(spark_session):
     assert compare.matches()
 
 
-@mock.patch("datacompy.spark.render")
-@mock.patch("datacompy.spark.save_html_report")
+@mock.patch("datacompy.report.render")
+@mock.patch("datacompy.base.save_html_report")
 def test_save_html(mock_save_html, mock_render, spark_session):
     df1 = spark_session.createDataFrame([{"a": 1, "b": 2}, {"a": 1, "b": 2}])
     df2 = spark_session.createDataFrame([{"a": 1, "b": 2}, {"a": 1, "b": 2}])
@@ -1976,8 +1976,8 @@ def test_template_context_variables(spark_session):
             os.unlink(template_path)
 
 
-@mock.patch("datacompy.spark.save_html_report")
-@mock.patch("datacompy.spark.render")
+@mock.patch("datacompy.base.save_html_report")
+@mock.patch("datacompy.report.render")
 def test_html_report_generation(mock_render, mock_save_html, spark_session):
     """Test that HTML reports can be generated and saved to a file."""
     df1 = spark_session.createDataFrame([("a", 1), ("b", 2)], ["id", "value"])