diff --git a/datapipe/store/database.py b/datapipe/store/database.py index 15f461f3..daf13645 100644 --- a/datapipe/store/database.py +++ b/datapipe/store/database.py @@ -3,7 +3,6 @@ import math from typing import Any, Callable, Iterator, cast -import numpy as np import pandas as pd from opentelemetry import trace from sqlalchemy import Column, MetaData, Table, create_engine, func, text @@ -239,10 +238,12 @@ def update_rows(self, df: DataDF) -> None: if df.empty: return - # TODO разобраться, нужен ли этот блок кода, написать тесты на заполнение None/NULL - insert_sql = self.dbconn.insert(self.data_table).values( - df.fillna(np.nan).replace({np.nan: None}).to_dict(orient="records") - ) + # SQLAlchemy expects Python None for SQL NULL; keep object dtype to avoid + # pandas downcasting warnings while preserving scalar values. + df_records = df.astype(object) + df_records[pd.isna(df_records)] = None + records = df_records.to_dict(orient="records") + insert_sql = self.dbconn.insert(self.data_table).values(records) if len(self.data_keys) > 0: sql = insert_sql.on_conflict_do_update( diff --git a/datapipe/store/table_store.py b/datapipe/store/table_store.py index 8d3d2839..8f3cefcc 100644 --- a/datapipe/store/table_store.py +++ b/datapipe/store/table_store.py @@ -45,7 +45,7 @@ def hash_keys(self) -> list[str]: return self.primary_keys + self.meta_keys def hash_rows(self, df: DataDF) -> HashDF: - hash_df = df[self.hash_keys] + hash_df = df[self.hash_keys].copy() hash_df["hash"] = df.apply(lambda x: str(list(x)), axis=1).apply( lambda x: int.from_bytes(cityhash.CityHash32(x).to_bytes(4, "little"), "little", signed=True) )