From a87408c19a543ff3fd95104b7280a708187217d6 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Tue, 30 Jun 2026 18:52:39 +0200 Subject: [PATCH 1/4] feat(mssql): add Microsoft Entra ID authentication methods across MS SQL destinations Add an `authentication` credential option to MsSqlCredentials so the mssql, synapse and fabric destinations can authenticate to Azure-hosted SQL with Microsoft Entra ID instead of a SQL login: - azure-identity token methods (cli, default/auto, environment, interactive, devicecode, msi/managedidentity): dlt acquires an access token and injects it via pyodbc attrs_before (SQL_COPT_SS_ACCESS_TOKEN), so they work cross-platform (including macOS, where the ODBC driver's built-in Entra ID modes are unreliable). - driver-native methods (ActiveDirectoryServicePrincipal, ActiveDirectoryPassword, ActiveDirectoryIntegrated, ActiveDirectoryInteractive) are passed as Authentication=. - empty authentication keeps the existing SQL login (backwards compatible); the default Service Principal method without a secret falls back to DefaultAzureCredential. The shared auth helpers live in the mssql destination; synapse inherits them and fabric reuses them, dropping its previous fabric-only token override. azure-identity is imported lazily (only for token methods). Adds unit tests for every method on mssql and fabric, a Docker-based integration test for the SQL-login connection path, and documentation. Closes #2059 Co-Authored-By: Claude Opus 4.8 --- dlt/destinations/impl/fabric/configuration.py | 83 ++++--- dlt/destinations/impl/fabric/sql_client.py | 2 + dlt/destinations/impl/mssql/configuration.py | 216 +++++++++++++++++- dlt/destinations/impl/mssql/sql_client.py | 17 +- .../docs/dlt-ecosystem/destinations/fabric.md | 54 ++++- .../docs/dlt-ecosystem/destinations/mssql.md | 45 ++++ .../dlt-ecosystem/destinations/synapse.md | 4 + .../load/fabric/test_fabric_configuration.py | 155 +++++++++++++ tests/load/mssql/docker-compose.yml | 18 ++ tests/load/mssql/test_mssql_configuration.py | 168 +++++++++++++- tests/load/mssql/test_mssql_docker_auth.py | 61 +++++ 11 files changed, 773 insertions(+), 50 deletions(-) create mode 100644 tests/load/mssql/docker-compose.yml create mode 100644 tests/load/mssql/test_mssql_docker_auth.py diff --git a/dlt/destinations/impl/fabric/configuration.py b/dlt/destinations/impl/fabric/configuration.py index 37c34a3582..ffc920fb3c 100644 --- a/dlt/destinations/impl/fabric/configuration.py +++ b/dlt/destinations/impl/fabric/configuration.py @@ -4,20 +4,37 @@ from dlt.common.configuration import configspec from dlt.common.configuration.specs import AzureServicePrincipalCredentials from dlt.common.destination.client import DestinationClientDwhWithStagingConfiguration -from dlt.common.exceptions import MissingDependencyException +from dlt.common.typing import TSecretStrValue from dlt.common.utils import digest128 -from dlt import version +from dlt.destinations.impl.mssql.configuration import ( + apply_authentication_to_dsn, + build_token_attrs_before, + setup_token_credential, + validate_authentication, +) -_AZURE_STORAGE_EXTRA = f"{version.DLT_PKG_NAME}[az]" +# Fabric Warehouse only supports Entra ID authentication, so it defaults to Service Principal +# (the shared MsSql auth machinery additionally enables azure-identity token methods). +_DEFAULT_AUTHENTICATION = "ActiveDirectoryServicePrincipal" @configspec(init=False) class FabricCredentials(AzureServicePrincipalCredentials): - """Credentials for Microsoft Fabric Warehouse with Service Principal authentication. + """Credentials for Microsoft Fabric Warehouse. - Fabric Warehouse requires Azure AD Service Principal authentication. - Inherits from AzureServicePrincipalCredentials for Service Principal fields and - automatic fallback to DefaultAzureCredential. + Supports several Entra ID authentication methods, selected through `authentication`: + + * **Driver-native** (the ODBC driver authenticates): `ActiveDirectoryServicePrincipal` + (default), `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, + `ActiveDirectoryInteractive`. + * **azure-identity** (dlt acquires an access token and injects it, works cross-platform): + `auto`/`default` (DefaultAzureCredential), `cli`, `environment`, `interactive`, + `devicecode`, `msi`/`managedidentity`. + + When `authentication` is left at its default but no Service Principal secret is configured, + dlt falls back to `DefaultAzureCredential` and injects its token. + + Inherits from AzureServicePrincipalCredentials for the Service Principal fields. """ drivername: str = "mssql+pyodbc" @@ -35,43 +52,55 @@ class FabricCredentials(AzureServicePrincipalCredentials): connect_timeout: int = 15 """Connection timeout in seconds (default: 15)""" + authentication: str = _DEFAULT_AUTHENTICATION + """Authentication method. Driver-native: `ActiveDirectoryServicePrincipal` (default), + `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`. + azure-identity (token injected by dlt): `auto`/`default`, `cli`, `environment`, + `interactive`, `devicecode`, `msi`/`managedidentity`.""" + + username: str | None = None + """User principal name, used with `ActiveDirectoryPassword` authentication.""" + + password: TSecretStrValue | None = None + """Password, used with `ActiveDirectoryPassword` authentication.""" + # Override to make optional - not needed for Fabric Warehouse credentials (only for staging) azure_storage_account_name: Optional[str] = None """Not used for Fabric Warehouse credentials (only staging credentials need this)""" def on_partial(self) -> None: - """Enable fallback to DefaultAzureCredential if explicit credentials not provided.""" - try: - from azure.identity import DefaultAzureCredential - except ModuleNotFoundError: - raise MissingDependencyException(self.__class__.__name__, [_AZURE_STORAGE_EXTRA]) - - # If no explicit Service Principal credentials, use default credentials - if not self.azure_client_id or not self.azure_client_secret or not self.azure_tenant_id: - self._set_default_credentials(DefaultAzureCredential()) - # Resolve if we have warehouse connection details (not storage account name) - if self.host and self.database: - self.resolve() + """Set up token-based credentials and resolve once host and database are known. + + Token-based methods (and the default Service Principal method without a secret) get an + azure-identity credential whose token is injected into the connection. Driver-native + methods need no token and resolve as-is. Auth logic is shared with `MsSqlCredentials`. + """ + setup_token_credential(self) + # Resolve if we have the warehouse connection details (not the storage account name) + if self.host and self.database: + self.resolve() + + def on_resolved(self) -> None: + """Validate the configured authentication method.""" + validate_authentication(self) def get_odbc_dsn_dict(self) -> Dict[str, Any]: """Build ODBC DSN dictionary with Fabric-specific settings.""" - params = { + params: dict[str, Any] = { "DRIVER": "{ODBC Driver 18 for SQL Server}", "SERVER": f"{self.host},{self.port}", "DATABASE": self.database, - "AUTHENTICATION": "ActiveDirectoryServicePrincipal", "LongAsMax": "yes", # Required for UTF-8 collation support "Encrypt": "yes", "TrustServerCertificate": "no", } - - # Add Service Principal credentials if provided - if self.azure_client_id and self.azure_tenant_id and self.azure_client_secret: - params["UID"] = f"{self.azure_client_id}@{self.azure_tenant_id}" - params["PWD"] = str(self.azure_client_secret) - + apply_authentication_to_dsn(self, params) return params + def to_odbc_attrs_before(self) -> dict[int, bytes] | None: + """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + return build_token_attrs_before(self) + def to_odbc_dsn(self) -> str: """Build ODBC connection string for pyodbc.""" params = self.get_odbc_dsn_dict() diff --git a/dlt/destinations/impl/fabric/sql_client.py b/dlt/destinations/impl/fabric/sql_client.py index c7309eab95..4fbfeac40a 100644 --- a/dlt/destinations/impl/fabric/sql_client.py +++ b/dlt/destinations/impl/fabric/sql_client.py @@ -13,6 +13,8 @@ class FabricSqlClient(SynapseSqlClient): """SQL client for Microsoft Fabric Warehouse Inherits all behavior from Synapse since Fabric Warehouse is built on Synapse technology. + Azure AD token injection (for token-based authentication) is handled by the shared + `PyOdbcMsSqlClient.open_connection`. """ def __init__( diff --git a/dlt/destinations/impl/mssql/configuration.py b/dlt/destinations/impl/mssql/configuration.py index b36a690f30..e7a6b17073 100644 --- a/dlt/destinations/impl/mssql/configuration.py +++ b/dlt/destinations/impl/mssql/configuration.py @@ -1,14 +1,187 @@ import dataclasses -from typing import ClassVar, Any, Final, List, Dict, Optional +import struct +from typing import ClassVar, Any, Final, List, Dict, Optional, TYPE_CHECKING +from dlt import version from dlt.common.configuration import configspec -from dlt.common.configuration.specs import ConnectionStringCredentials +from dlt.common.configuration.exceptions import ConfigurationException +from dlt.common.configuration.specs import ConnectionStringCredentials, CredentialsWithDefault from dlt.common.typing import TSecretStrValue -from dlt.common.exceptions import SystemConfigurationException +from dlt.common.exceptions import MissingDependencyException, SystemConfigurationException from dlt.common.destination.client import DestinationClientDwhWithStagingConfiguration from dlt.common.utils import digest128 +if TYPE_CHECKING: + from azure.core.credentials import TokenCredential + +_AZURE_AUTH_EXTRA = f"{version.DLT_PKG_NAME}[az]" + +# pyodbc connection attribute used to inject a pre-acquired Entra ID access token. +# https://learn.microsoft.com/sql/connect/odbc/using-azure-active-directory#authenticating-with-an-access-token +SQL_COPT_SS_ACCESS_TOKEN = 1256 +SQL_TOKEN_SCOPE = "https://database.windows.net/.default" + +# Authentication methods handled by the ODBC driver itself (passed as `Authentication=` in the DSN). +DRIVER_NATIVE_AUTHENTICATION = frozenset( + { + "ActiveDirectoryServicePrincipal", + "ActiveDirectoryPassword", + "ActiveDirectoryIntegrated", + "ActiveDirectoryInteractive", + } +) + +# Authentication methods backed by azure-identity. dlt acquires the access token and injects it +# into the connection, so these work cross-platform without relying on ODBC driver support. +AZURE_IDENTITY_AUTHENTICATION = frozenset( + { + "auto", + "default", + "cli", + "environment", + "interactive", + "devicecode", + "msi", + "managedidentity", + } +) + + +def create_token_credential(authentication: str) -> "TokenCredential": + """Create an azure-identity credential for a token-based authentication method.""" + try: + from azure.identity import ( + AzureCliCredential, + DefaultAzureCredential, + DeviceCodeCredential, + EnvironmentCredential, + InteractiveBrowserCredential, + ManagedIdentityCredential, + ) + except ModuleNotFoundError: + raise MissingDependencyException("MsSqlCredentials", [_AZURE_AUTH_EXTRA]) + + credential_factories = { + "auto": DefaultAzureCredential, + "default": DefaultAzureCredential, + "cli": AzureCliCredential, + "environment": EnvironmentCredential, + "interactive": InteractiveBrowserCredential, + "devicecode": DeviceCodeCredential, + "msi": ManagedIdentityCredential, + "managedidentity": ManagedIdentityCredential, + } + return credential_factories[authentication.lower()]() # type: ignore[no-any-return] + + +def uses_token_authentication(credentials: Any) -> bool: + """True when dlt must acquire an access token and inject it into the connection. + + Derived from the configured method, not from `has_default_credentials`: cooperative + `on_partial` calls up the MRO may set a default credential even for driver-native methods. + """ + authentication = credentials.authentication + if not authentication: + return False + if authentication.lower() in AZURE_IDENTITY_AUTHENTICATION: + return True + # ActiveDirectoryServicePrincipal without a secret cannot authenticate through the ODBC + # driver, so dlt falls back to a DefaultAzureCredential token. + return authentication == "ActiveDirectoryServicePrincipal" and not ( + credentials.azure_client_id + and credentials.azure_client_secret + and credentials.azure_tenant_id + ) + + +def setup_token_credential(credentials: Any) -> None: + """Create and store the azure-identity credential for token-based authentication.""" + authentication = credentials.authentication or "" + if authentication.lower() in AZURE_IDENTITY_AUTHENTICATION: + credentials._set_default_credentials(create_token_credential(authentication)) + elif authentication == "ActiveDirectoryServicePrincipal" and not ( + credentials.azure_client_id + and credentials.azure_client_secret + and credentials.azure_tenant_id + ): + credentials._set_default_credentials(create_token_credential("default")) + + +def get_token_credential(credentials: Any) -> "TokenCredential": + """Return the azure-identity credential used for token authentication. + + Reuses the credential created during resolution (so azure-identity can cache tokens + across connections) and creates one on demand otherwise. + """ + if credentials.has_default_credentials(): + return credentials.default_credentials() # type: ignore[no-any-return] + authentication = credentials.authentication or "" + if authentication.lower() not in AZURE_IDENTITY_AUTHENTICATION: + authentication = "default" + return create_token_credential(authentication) + + +def build_token_attrs_before(credentials: Any) -> dict[int, bytes] | None: + """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + if not uses_token_authentication(credentials): + return None + token = get_token_credential(credentials).get_token(SQL_TOKEN_SCOPE).token + encoded_token = token.encode("utf-16-le") + token_struct = struct.pack(f" None: + """Validate the configured authentication method.""" + authentication = credentials.authentication + if not authentication: + return # plain SQL login (username/password) + if ( + authentication not in DRIVER_NATIVE_AUTHENTICATION + and authentication.lower() not in AZURE_IDENTITY_AUTHENTICATION + ): + supported = sorted(DRIVER_NATIVE_AUTHENTICATION) + sorted(AZURE_IDENTITY_AUTHENTICATION) + raise ConfigurationException( + f"Unsupported `authentication` method `{authentication}`." + f" Supported methods: {', '.join(supported)}." + ) + if authentication == "ActiveDirectoryPassword" and not ( + credentials.username and credentials.password + ): + raise ConfigurationException( + "`authentication = ActiveDirectoryPassword` requires `username` and `password`." + ) + + +def apply_authentication_to_dsn(credentials: Any, params: dict[str, Any]) -> None: + """Add UID/PWD/Authentication keys to an ODBC DSN dict based on the authentication method.""" + if uses_token_authentication(credentials): + # The access token is injected via attrs_before, so the DSN carries no credentials. + return + authentication = credentials.authentication + if not authentication: + # Plain SQL login. + params["UID"] = credentials.username + params["PWD"] = credentials.password + return + params["AUTHENTICATION"] = authentication + if ( + authentication == "ActiveDirectoryServicePrincipal" + and credentials.azure_client_id + and credentials.azure_tenant_id + and credentials.azure_client_secret + ): + params["UID"] = f"{credentials.azure_client_id}@{credentials.azure_tenant_id}" + params["PWD"] = str(credentials.azure_client_secret) + elif ( + authentication == "ActiveDirectoryPassword" + and credentials.username + and credentials.password + ): + params["UID"] = credentials.username + params["PWD"] = credentials.password + def escape_mssql_odbc_value(value: Optional[str]) -> str: """Escape a value for MSSQL ADO/ODBC connection string format. @@ -50,7 +223,7 @@ def build_odbc_dsn(params: Dict[str, Any]) -> str: @configspec(init=False) -class MsSqlCredentials(ConnectionStringCredentials): +class MsSqlCredentials(ConnectionStringCredentials, CredentialsWithDefault): drivername: Final[str] = dataclasses.field(default="mssql", init=False, repr=False, compare=False) # type: ignore[misc] database: str = None username: str = None @@ -60,6 +233,22 @@ class MsSqlCredentials(ConnectionStringCredentials): connect_timeout: int = 30 driver: str = None + authentication: str | None = None + """Authentication method. Empty (default) uses plain SQL login (`username`/`password`). + Driver-native: `ActiveDirectoryServicePrincipal`, `ActiveDirectoryPassword`, + `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`. azure-identity (token injected by + dlt): `auto`/`default`, `cli`, `environment`, `interactive`, `devicecode`, + `msi`/`managedidentity`.""" + + azure_tenant_id: str | None = None + """Entra ID tenant id, used with `ActiveDirectoryServicePrincipal` authentication.""" + + azure_client_id: str | None = None + """Service Principal client id, used with `ActiveDirectoryServicePrincipal` authentication.""" + + azure_client_secret: TSecretStrValue | None = None + """Service Principal client secret, used with `ActiveDirectoryServicePrincipal` authentication.""" + __config_gen_annotations__: ClassVar[List[str]] = ["port", "connect_timeout"] SUPPORTED_DRIVERS: ClassVar[List[str]] = [ @@ -76,6 +265,7 @@ def parse_native_representation(self, native_value: Any) -> None: self.connect_timeout = int(self.query.get("connect_timeout", self.connect_timeout)) def on_resolved(self) -> None: + validate_authentication(self) if self.driver not in self.SUPPORTED_DRIVERS: raise SystemConfigurationException( f"The specified driver `{self.driver}` is not supported." @@ -90,7 +280,14 @@ def get_query(self) -> Dict[str, Any]: def on_partial(self) -> None: self.driver = self._get_driver() - if not self.is_partial(): + setup_token_credential(self) + if self.authentication: + # Entra ID methods (token or driver-native) supply their own credentials and do not + # rely on username/password; resolve once we have a target. `on_resolved` validates. + if self.host and self.database: + self.resolve() + elif not self.is_partial(): + # Plain SQL login needs username/password. self.resolve() def _get_driver(self) -> str: @@ -111,13 +308,12 @@ def _get_driver(self) -> str: ) def get_odbc_dsn_dict(self) -> Dict[str, Any]: - params = { + params: dict[str, Any] = { "DRIVER": self.driver, "SERVER": f"{self.host},{self.port}", "DATABASE": self.database, - "UID": self.username, - "PWD": self.password, } + apply_authentication_to_dsn(self, params) if self.query is not None: params.update({k.upper(): v for k, v in self.query.items()}) return params @@ -126,6 +322,10 @@ def to_odbc_dsn(self) -> str: params = self.get_odbc_dsn_dict() return build_odbc_dsn(params) + def to_odbc_attrs_before(self) -> dict[int, bytes] | None: + """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + return build_token_attrs_before(self) + @configspec class MsSqlClientConfiguration(DestinationClientDwhWithStagingConfiguration): diff --git a/dlt/destinations/impl/mssql/sql_client.py b/dlt/destinations/impl/mssql/sql_client.py index c06122a802..ab5d9e6fdd 100644 --- a/dlt/destinations/impl/mssql/sql_client.py +++ b/dlt/destinations/impl/mssql/sql_client.py @@ -55,10 +55,19 @@ def __init__( self.credentials = credentials def open_connection(self) -> pyodbc.Connection: - self._conn = pyodbc.connect( - self.credentials.to_odbc_dsn(), - timeout=self.credentials.connect_timeout, - ) + # For Entra ID token authentication, inject a fresh access token via attrs_before. + attrs_before = self.credentials.to_odbc_attrs_before() + if attrs_before: + self._conn = pyodbc.connect( + self.credentials.to_odbc_dsn(), + timeout=self.credentials.connect_timeout, + attrs_before=attrs_before, + ) + else: + self._conn = pyodbc.connect( + self.credentials.to_odbc_dsn(), + timeout=self.credentials.connect_timeout, + ) # https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function self._conn.add_output_converter(-155, handle_datetimeoffset) self._conn.autocommit = True diff --git a/docs/website/docs/dlt-ecosystem/destinations/fabric.md b/docs/website/docs/dlt-ecosystem/destinations/fabric.md index 48cf16c7c4..1cd53ce5bd 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/fabric.md +++ b/docs/website/docs/dlt-ecosystem/destinations/fabric.md @@ -29,21 +29,44 @@ Supported driver versions: You can also [configure the driver name](#additional-destination-options) explicitly. -### Service Principal Authentication +### Authentication -Fabric Warehouse requires Azure Active Directory Service Principal authentication. You'll need: - -1. **Tenant ID**: Your Azure AD tenant ID (GUID) -2. **Client ID**: Application (service principal) client ID (GUID) -3. **Client Secret**: Application client secret -4. **Host**: Your Fabric warehouse SQL endpoint -5. **Database**: The database name within your warehouse +Fabric Warehouse authenticates with Microsoft Entra ID. Whichever method you choose, you always set the warehouse SQL endpoint as the `host` (`.datawarehouse.fabric.microsoft.com`) and the warehouse name as the `database`. **Finding your SQL endpoint:** - In the Fabric portal, go to your warehouse **Settings** - Select **SQL endpoint** - Copy the **SQL connection string** - it should be in the format: `.datawarehouse.fabric.microsoft.com` +The authentication method is selected with the `authentication` credential option. `dlt` supports +two families of methods. + +With the **driver-native** methods, the ODBC driver performs the Entra ID sign-in: + +| `authentication` | Description | Required fields | +|---|---|---| +| `ActiveDirectoryServicePrincipal` (default) | Service Principal | `azure_tenant_id`, `azure_client_id`, `azure_client_secret` | +| `ActiveDirectoryPassword` | Entra ID username/password | `username`, `password` | +| `ActiveDirectoryIntegrated` | Integrated Windows authentication | None | +| `ActiveDirectoryInteractive` | Interactive browser prompt (driver) | None | + +With the **azure-identity** methods, `dlt` acquires an access token with +[azure-identity](https://learn.microsoft.com/python/api/overview/azure/identity-readme) and injects +it into the connection. These work cross-platform (including macOS, where the ODBC driver's built-in +Entra ID modes are unreliable) and need no secret in `secrets.toml`: + +| `authentication` | azure-identity credential | +|---|---| +| `auto` / `default` | `DefaultAzureCredential` | +| `cli` | `AzureCliCredential` (uses `az login`) | +| `environment` | `EnvironmentCredential` | +| `interactive` | `InteractiveBrowserCredential` | +| `devicecode` | `DeviceCodeCredential` | +| `msi` / `managedidentity` | `ManagedIdentityCredential` | + +When `authentication` is left at its default but no Service Principal secret is configured, `dlt` +falls back to `DefaultAzureCredential`. + ### Create a pipeline **1. Initialize a project with a pipeline that loads to Fabric by running:** @@ -62,6 +85,8 @@ pip install "dlt[fabric]" **3. Enter your credentials into `.dlt/secrets.toml`.** +Service Principal (default): + ```toml [destination.fabric.credentials] host = ".datawarehouse.fabric.microsoft.com" @@ -73,6 +98,15 @@ port = 1433 connect_timeout = 30 ``` +azure-identity, e.g. the Azure CLI (`az login`), which needs no secret: + +```toml +[destination.fabric.credentials] +host = ".datawarehouse.fabric.microsoft.com" +database = "mydb" +authentication = "cli" +``` + ## Write disposition All write dispositions are supported, including the [`upsert`](../../general-usage/merge-loading.md#upsert-strategy) and [`insert-only`](../../general-usage/merge-loading.md#insert-only-strategy) merge strategies. @@ -205,7 +239,7 @@ driver="ODBC Driver 18 for SQL Server" While Fabric Warehouse is based on SQL Server, there are key differences: -1. **Authentication**: Fabric requires Service Principal; username/password auth is not supported +1. **Authentication**: Fabric uses Entra ID; in addition to Service Principal, `dlt` supports several azure-identity methods (see [Authentication](#authentication)) 2. **Type System**: Uses `varchar` and `datetime2` instead of `nvarchar` and `datetimeoffset` 3. **Collation**: Optimized for UTF-8 collations with automatic `LongAsMax` configuration 4. **SQL Dialect**: Uses `fabric` SQLglot dialect for proper SQL generation @@ -232,7 +266,7 @@ sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 Ensure your Service Principal has: - Proper permissions on the Fabric workspace - Access to the target database/warehouse -- Correct tenant ID (your Azure AD tenant, not the workspace/capacity ID) +- Correct tenant ID (your Entra ID tenant, not the workspace/capacity ID) ### UTF-8 Character Issues diff --git a/docs/website/docs/dlt-ecosystem/destinations/mssql.md b/docs/website/docs/dlt-ecosystem/destinations/mssql.md index 82aa70c86a..8af58e915d 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/mssql.md +++ b/docs/website/docs/dlt-ecosystem/destinations/mssql.md @@ -97,6 +97,51 @@ destination.mssql.credentials="mssql://loader:loader@localhost/dlt_data?TrustSer destination.mssql.credentials="mssql://loader:loader@localhost/dlt_data?LongAsMax=yes" ``` +### Microsoft Entra ID authentication + +For Azure-hosted SQL Server (Azure SQL Database, Managed Instance) you can authenticate with +Entra ID instead of a SQL login. Set the `authentication` credential option. + +With the **azure-identity** methods, `dlt` acquires an access token and injects it into the +connection, so they work cross-platform (including macOS) and need no password in `secrets.toml`. +They require the `azure-identity` package (installed with `pip install "dlt[az]"`). + +| `authentication` | How it authenticates | +|---|---| +| _(empty, default)_ | SQL login with `username`/`password` | +| `cli` | `AzureCliCredential` (uses `az login`) | +| `auto` / `default` | `DefaultAzureCredential` (managed identity, env, CLI, …) | +| `environment` | `EnvironmentCredential` | +| `interactive` | `InteractiveBrowserCredential` | +| `devicecode` | `DeviceCodeCredential` | +| `msi` / `managedidentity` | `ManagedIdentityCredential` | +| `ActiveDirectoryServicePrincipal` | Service Principal (`azure_tenant_id`, `azure_client_id`, `azure_client_secret`) | +| `ActiveDirectoryPassword` | Entra ID `username`/`password` (handled by the ODBC driver) | +| `ActiveDirectoryIntegrated` | Integrated Windows authentication (handled by the ODBC driver) | +| `ActiveDirectoryInteractive` | Interactive prompt (handled by the ODBC driver) | + +Passwordless example using the Azure CLI: +```toml +[destination.mssql.credentials] +database = "dlt_data" +host = "loader.database.windows.net" +authentication = "cli" +``` + +Service Principal example: +```toml +[destination.mssql.credentials] +database = "dlt_data" +host = "loader.database.windows.net" +authentication = "ActiveDirectoryServicePrincipal" +azure_tenant_id = "your-tenant-id" +azure_client_id = "your-client-id" +azure_client_secret = "your-client-secret" +``` + +When `authentication` is left empty but no `password` is set, `dlt` falls back to +`DefaultAzureCredential`. + **To pass credentials directly**, use the [explicit instance of the destination](../../general-usage/destination.md#pass-explicit-credentials) ```py pipeline = dlt.pipeline( diff --git a/docs/website/docs/dlt-ecosystem/destinations/synapse.md b/docs/website/docs/dlt-ecosystem/destinations/synapse.md index 08e14eb094..a0e3ae493e 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/synapse.md +++ b/docs/website/docs/dlt-ecosystem/destinations/synapse.md @@ -97,6 +97,10 @@ pipeline = dlt.pipeline( dataset_name='chess_data' ) ``` +**Microsoft Entra ID authentication** is supported through the same `authentication` credential +option as the [mssql destination](./mssql.md#microsoft-entra-id-authentication), including the +passwordless azure-identity methods (`cli`, `default`, `msi`, …) that inject an access token. + To use **Active Directory Principal**, you can use the `sqlalchemy.engine.URL.create` method to create the connection URL using your Active Directory Service Principal credentials. First, create the connection string as: ```py conn_str = ( diff --git a/tests/load/fabric/test_fabric_configuration.py b/tests/load/fabric/test_fabric_configuration.py index e69c71bde7..19897a6dec 100644 --- a/tests/load/fabric/test_fabric_configuration.py +++ b/tests/load/fabric/test_fabric_configuration.py @@ -1,11 +1,13 @@ """Tests for Microsoft Fabric Warehouse destination configuration""" import os +import struct from typing import Optional import pytest from dlt.common.configuration import resolve_configuration +from dlt.common.configuration.exceptions import ConfigurationException from dlt.common.schema import Schema from dlt.common.utils import digest128 from dlt.destinations.impl.fabric.factory import fabric @@ -13,6 +15,7 @@ FabricCredentials, FabricClientConfiguration, ) +from dlt.destinations.impl.mssql.configuration import uses_token_authentication # mark all tests as essential, do not remove pytestmark = pytest.mark.essential @@ -221,3 +224,155 @@ def test_fabric_credentials_authentication_method() -> None: # Verify ActiveDirectoryServicePrincipal is set dsn_dict = creds.get_odbc_dsn_dict() assert dsn_dict["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + + +# --------------------------------------------------------------------------- +# Authentication methods +# --------------------------------------------------------------------------- + + +class _FakeAccessToken: + token = "fake-access-token" + + +class _FakeTokenCredential: + """Minimal azure-identity-like credential, avoids hitting Azure in unit tests.""" + + def get_token(self, *scopes: str, **kwargs: object) -> _FakeAccessToken: + return _FakeAccessToken() + + +def _warehouse_credentials( + authentication: str | None = None, **kwargs: object +) -> FabricCredentials: + creds = FabricCredentials() + creds.host = "test.datawarehouse.fabric.microsoft.com" + creds.database = "testdb" + if authentication is not None: + creds.authentication = authentication + for key, value in kwargs.items(): + setattr(creds, key, value) + return creds + + +def test_fabric_authentication_default_is_service_principal() -> None: + assert FabricCredentials().authentication == "ActiveDirectoryServicePrincipal" + + +@pytest.mark.parametrize( + "authentication,expected", + [ + ("auto", "DefaultAzureCredential"), + ("default", "DefaultAzureCredential"), + ("cli", "AzureCliCredential"), + ("environment", "EnvironmentCredential"), + ("interactive", "InteractiveBrowserCredential"), + ("devicecode", "DeviceCodeCredential"), + ("msi", "ManagedIdentityCredential"), + ("managedidentity", "ManagedIdentityCredential"), + ], +) +def test_fabric_azure_identity_credential_mapping(authentication: str, expected: str) -> None: + """Each azure-identity method maps to the right credential and skips the DSN AUTHENTICATION.""" + creds = _warehouse_credentials(authentication) + creds.on_partial() + + assert type(creds.default_credentials()).__name__ == expected + + dsn = creds.get_odbc_dsn_dict() + assert "AUTHENTICATION" not in dsn + assert "UID" not in dsn + assert "PWD" not in dsn + assert dsn["LongAsMax"] == "yes" + + +def test_fabric_service_principal_without_secret_falls_back_to_token() -> None: + """Default method without a Service Principal secret injects a DefaultAzureCredential token.""" + creds = _warehouse_credentials() + creds.on_partial() + + assert uses_token_authentication(creds) is True + assert type(creds.default_credentials()).__name__ == "DefaultAzureCredential" + assert "AUTHENTICATION" not in creds.get_odbc_dsn_dict() + + +def test_fabric_service_principal_with_secret_is_driver_native() -> None: + """Default method with a Service Principal secret authenticates through the ODBC driver.""" + creds = _warehouse_credentials( + azure_tenant_id="t", azure_client_id="c", azure_client_secret="s" + ) + creds.on_partial() + + assert uses_token_authentication(creds) is False + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + assert dsn["UID"] == "c@t" + assert dsn["PWD"] == "s" + assert creds.to_odbc_attrs_before() is None + + +@pytest.mark.parametrize( + "authentication", ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive"] +) +def test_fabric_driver_native_passthrough(authentication: str) -> None: + creds = _warehouse_credentials(authentication) + creds.on_partial() + + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == authentication + assert "UID" not in dsn + assert "PWD" not in dsn + assert creds.to_odbc_attrs_before() is None + + +def test_fabric_active_directory_password() -> None: + creds = _warehouse_credentials( + "ActiveDirectoryPassword", username="user@contoso.com", password="pwd" + ) + creds.on_partial() + + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryPassword" + assert dsn["UID"] == "user@contoso.com" + assert dsn["PWD"] == "pwd" + assert creds.to_odbc_attrs_before() is None + + +def test_fabric_active_directory_password_requires_username_password() -> None: + creds = _warehouse_credentials("ActiveDirectoryPassword") + with pytest.raises(ConfigurationException): + creds.on_partial() # on_partial -> resolve() -> on_resolved validates + + +def test_fabric_unsupported_authentication_raises() -> None: + creds = _warehouse_credentials("SqlPassword") + with pytest.raises(ConfigurationException): + creds.on_partial() + + +def test_fabric_to_odbc_attrs_before_token_struct() -> None: + """The injected token follows the SQL_COPT_SS_ACCESS_TOKEN struct layout.""" + creds = _warehouse_credentials("cli") + creds._set_default_credentials(_FakeTokenCredential()) + + attrs = creds.to_odbc_attrs_before() + assert attrs is not None + token_struct = attrs[1256] # SQL_COPT_SS_ACCESS_TOKEN + length = struct.unpack(" None: + """Resolution succeeds without a Service Principal secret for token authentication.""" + creds = FabricCredentials() + creds.host = "abc.datawarehouse.fabric.microsoft.com" + creds.database = "mydb" + creds.authentication = "cli" + + resolved = resolve_configuration(creds) + + assert resolved.is_resolved() + assert uses_token_authentication(resolved) is True + assert type(resolved.default_credentials()).__name__ == "AzureCliCredential" + assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() diff --git a/tests/load/mssql/docker-compose.yml b/tests/load/mssql/docker-compose.yml new file mode 100644 index 0000000000..44659559e4 --- /dev/null +++ b/tests/load/mssql/docker-compose.yml @@ -0,0 +1,18 @@ +# Local SQL Server for mssql destination integration tests. +# Start with `docker compose up`; tests skip automatically when the server is unreachable. +# +# Uses SQL Server 2025, which is required because the destination maps the dlt `json` type to +# the native `json` column type that older SQL Server versions do not support. +# On Apple Silicon, enable Rosetta in Docker Desktop so the amd64 image runs under emulation. +services: + mssql: + image: mcr.microsoft.com/mssql/server:2025-latest + container_name: dlt_test_mssql + restart: on-failure + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "Strong!Passw0rd" + MSSQL_PID: "Developer" + ports: + - 1433:1433 + # (this is just an example, not intended to be a production configuration) diff --git a/tests/load/mssql/test_mssql_configuration.py b/tests/load/mssql/test_mssql_configuration.py index d261391edc..81aa1411b7 100644 --- a/tests/load/mssql/test_mssql_configuration.py +++ b/tests/load/mssql/test_mssql_configuration.py @@ -1,14 +1,21 @@ import os +import struct import pyodbc import pytest from dlt.common.configuration import ConfigFieldMissingException, resolve_configuration +from dlt.common.configuration.exceptions import ConfigurationException from dlt.common.exceptions import SystemConfigurationException from dlt.common.schema import Schema from dlt.common.utils import digest128 from dlt.destinations import mssql -from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration, MsSqlCredentials +from dlt.destinations.impl.mssql.configuration import ( + MsSqlClientConfiguration, + MsSqlCredentials, + uses_token_authentication, + validate_authentication, +) # mark all tests as essential, do not remove pytestmark = pytest.mark.essential @@ -208,3 +215,162 @@ def test_to_odbc_dsn_driver_not_specified() -> None: } for d in MsSqlCredentials.SUPPORTED_DRIVERS ] + + +# --------------------------------------------------------------------------- +# Authentication methods +# --------------------------------------------------------------------------- + + +class _FakeAccessToken: + token = "fake-access-token" + + +class _FakeTokenCredential: + """Minimal azure-identity-like credential, avoids hitting Azure in unit tests.""" + + def get_token(self, *scopes: str, **kwargs: object) -> _FakeAccessToken: + return _FakeAccessToken() + + +def _mssql_credentials(authentication: object = None, **kwargs: object) -> MsSqlCredentials: + creds = MsSqlCredentials() + creds.host = "sql.example.com" + creds.database = "test_db" + creds.driver = "ODBC Driver 18 for SQL Server" # avoid probing for an installed driver + if authentication is not None: + creds.authentication = authentication # type: ignore[assignment] + for key, value in kwargs.items(): + setattr(creds, key, value) + return creds + + +def test_mssql_authentication_defaults_to_sql_login() -> None: + assert MsSqlCredentials().authentication is None + + +def test_mssql_sql_login_dsn_uses_uid_pwd() -> None: + creds = _mssql_credentials(username="loader", password="secret") + creds.on_partial() + + dsn = creds.get_odbc_dsn_dict() + assert "AUTHENTICATION" not in dsn + assert dsn["UID"] == "loader" + assert dsn["PWD"] == "secret" + assert creds.to_odbc_attrs_before() is None + + +@pytest.mark.parametrize( + "authentication,expected", + [ + ("auto", "DefaultAzureCredential"), + ("default", "DefaultAzureCredential"), + ("cli", "AzureCliCredential"), + ("environment", "EnvironmentCredential"), + ("interactive", "InteractiveBrowserCredential"), + ("devicecode", "DeviceCodeCredential"), + ("msi", "ManagedIdentityCredential"), + ("managedidentity", "ManagedIdentityCredential"), + ], +) +def test_mssql_azure_identity_credential_mapping(authentication: str, expected: str) -> None: + creds = _mssql_credentials(authentication) + creds.on_partial() + + assert type(creds.default_credentials()).__name__ == expected + + dsn = creds.get_odbc_dsn_dict() + assert "AUTHENTICATION" not in dsn + assert "UID" not in dsn + assert "PWD" not in dsn + + +def test_mssql_service_principal_driver_native() -> None: + creds = _mssql_credentials( + "ActiveDirectoryServicePrincipal", + azure_tenant_id="t", + azure_client_id="c", + azure_client_secret="s", + ) + creds.on_partial() + + assert uses_token_authentication(creds) is False + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + assert dsn["UID"] == "c@t" + assert dsn["PWD"] == "s" + assert creds.to_odbc_attrs_before() is None + + +def test_mssql_service_principal_without_secret_falls_back_to_token() -> None: + creds = _mssql_credentials("ActiveDirectoryServicePrincipal") + creds.on_partial() + + assert uses_token_authentication(creds) is True + assert type(creds.default_credentials()).__name__ == "DefaultAzureCredential" + assert "AUTHENTICATION" not in creds.get_odbc_dsn_dict() + + +@pytest.mark.parametrize( + "authentication", ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive"] +) +def test_mssql_driver_native_passthrough(authentication: str) -> None: + creds = _mssql_credentials(authentication) + creds.on_partial() + + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == authentication + assert "UID" not in dsn + assert "PWD" not in dsn + assert creds.to_odbc_attrs_before() is None + + +def test_mssql_active_directory_password() -> None: + creds = _mssql_credentials( + "ActiveDirectoryPassword", username="user@contoso.com", password="pwd" + ) + creds.on_partial() + + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryPassword" + assert dsn["UID"] == "user@contoso.com" + assert dsn["PWD"] == "pwd" + + +def test_mssql_active_directory_password_requires_username_password() -> None: + creds = _mssql_credentials("ActiveDirectoryPassword") + with pytest.raises(ConfigurationException): + validate_authentication(creds) + + +def test_mssql_unsupported_authentication_raises() -> None: + creds = _mssql_credentials("SqlPassword", username="u", password="p") + with pytest.raises(ConfigurationException): + creds.on_partial() # resolves (all present) -> on_resolved -> validate raises + + +def test_mssql_to_odbc_attrs_before_token_struct() -> None: + creds = _mssql_credentials("cli") + creds._set_default_credentials(_FakeTokenCredential()) + + attrs = creds.to_odbc_attrs_before() + assert attrs is not None + token_struct = attrs[1256] # SQL_COPT_SS_ACCESS_TOKEN + length = struct.unpack(" None: + creds = MsSqlCredentials() + creds.host = "sql.example.com" + creds.database = "test_db" + creds.driver = "ODBC Driver 18 for SQL Server" + creds.authentication = "cli" + + resolved = resolve_configuration(creds) + + assert resolved.is_resolved() + assert uses_token_authentication(resolved) is True + assert type(resolved.default_credentials()).__name__ == "AzureCliCredential" + assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() diff --git a/tests/load/mssql/test_mssql_docker_auth.py b/tests/load/mssql/test_mssql_docker_auth.py new file mode 100644 index 0000000000..5a3f196566 --- /dev/null +++ b/tests/load/mssql/test_mssql_docker_auth.py @@ -0,0 +1,61 @@ +"""Integration tests for the mssql destination against a local SQL Server. + +Start the server with `docker compose up` in this directory (see docker-compose.yml). +Tests skip automatically when the server is unreachable, so they are safe to run anywhere. + +These exercise the real `PyOdbcMsSqlClient.open_connection` code path. SQL Server in Docker +only supports SQL login (no Entra ID), so token/Azure AD authentication is validated against a +real Azure SQL / Fabric instance instead, via the parametrized `tests/load` suite. +""" + +import pytest + +from dlt.common.configuration import resolve_configuration +from dlt.common.schema import Schema +from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration, MsSqlCredentials + +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + +DOCKER_HOST = "localhost" +DOCKER_DATABASE = "master" +DOCKER_USERNAME = "sa" +DOCKER_PASSWORD = "Strong!Passw0rd" + + +def _docker_credentials() -> MsSqlCredentials: + creds = MsSqlCredentials() + creds.host = DOCKER_HOST + creds.database = DOCKER_DATABASE + creds.username = DOCKER_USERNAME + creds.password = DOCKER_PASSWORD + creds.connect_timeout = 5 + # local container uses a self-signed certificate + creds.query = {"trustservercertificate": "yes", "encrypt": "no"} + return resolve_configuration(creds) + + +def _sql_client(creds: MsSqlCredentials): + from dlt.destinations import mssql + + config = MsSqlClientConfiguration(credentials=creds)._bind_dataset_name("dataset") + client = mssql().client(Schema("schema"), config) + return client.sql_client + + +def test_mssql_docker_sql_login_opens_connection() -> None: + """SQL login (no Azure AD) opens a real connection - attrs_before is skipped.""" + creds = _docker_credentials() + assert creds.to_odbc_attrs_before() is None + + sql_client = _sql_client(creds) + try: + sql_client.open_connection() + except Exception as ex: # noqa: BLE001 + pytest.skip(f"Local SQL Server not reachable (run `docker compose up`): {ex}") + + try: + rows = sql_client.execute_sql("SELECT 1") + assert rows[0][0] == 1 + finally: + sql_client.close_connection() From 4c12823b91ae779e9c5414d919946fcf39123804 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Wed, 1 Jul 2026 23:03:33 +0200 Subject: [PATCH 2/4] feat(mssql): use driver-native ODBC auth names, narrow token injection to two methods Redesign the `authentication` credential option to be driver-aware for the pyodbc driver used at this point in the mssql/synapse/fabric destinations: - Driver-native (passed as `Authentication=` in the DSN, poolable): the ODBC driver itself performs the sign-in. Adds `ActiveDirectoryMsi` alongside the existing `ActiveDirectoryServicePrincipal`, `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`. - azure-identity token injection (via pyodbc `attrs_before`, not poolable): narrowed to the two methods pyodbc/msodbcsql cannot perform natively, renamed to match ODBC's own naming convention: `ActiveDirectoryDefault` (DefaultAzureCredential) and `ActiveDirectoryDeviceCode` (DeviceCodeCredential). - The thin `default` alias for `ActiveDirectoryDefault` is kept; `auto` is not carried over since it never existed on `devel` and has no backwards-compatibility reason to exist here (PR1 would have been the one introducing it). - Removes the other dlt-custom lowercase names (`cli`, `environment`, `interactive`, `devicecode`, `msi`, `managedidentity`): the native `ActiveDirectory*` names now cover Interactive/DeviceCode/Msi directly, and `cli`/`environment` are covered by `ActiveDirectoryDefault`, whose `DefaultAzureCredential` chain already includes the Azure CLI and environment credentials. Updates the mssql/fabric/synapse destination docs and the offline mssql/fabric configuration unit tests to match. --- dlt/destinations/impl/fabric/configuration.py | 14 +-- dlt/destinations/impl/mssql/configuration.py | 94 ++++++++++--------- .../docs/dlt-ecosystem/destinations/fabric.md | 15 ++- .../docs/dlt-ecosystem/destinations/mssql.md | 15 ++- .../dlt-ecosystem/destinations/synapse.md | 3 +- .../load/fabric/test_fabric_configuration.py | 29 +++--- tests/load/mssql/test_mssql_configuration.py | 29 +++--- 7 files changed, 106 insertions(+), 93 deletions(-) diff --git a/dlt/destinations/impl/fabric/configuration.py b/dlt/destinations/impl/fabric/configuration.py index ffc920fb3c..c74bd993ce 100644 --- a/dlt/destinations/impl/fabric/configuration.py +++ b/dlt/destinations/impl/fabric/configuration.py @@ -26,13 +26,13 @@ class FabricCredentials(AzureServicePrincipalCredentials): * **Driver-native** (the ODBC driver authenticates): `ActiveDirectoryServicePrincipal` (default), `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, - `ActiveDirectoryInteractive`. + `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`. * **azure-identity** (dlt acquires an access token and injects it, works cross-platform): - `auto`/`default` (DefaultAzureCredential), `cli`, `environment`, `interactive`, - `devicecode`, `msi`/`managedidentity`. + `ActiveDirectoryDefault` (alias `default`, uses `DefaultAzureCredential`), + `ActiveDirectoryDeviceCode` (uses `DeviceCodeCredential`). When `authentication` is left at its default but no Service Principal secret is configured, - dlt falls back to `DefaultAzureCredential` and injects its token. + dlt falls back to `ActiveDirectoryDefault` and injects its token. Inherits from AzureServicePrincipalCredentials for the Service Principal fields. """ @@ -54,9 +54,9 @@ class FabricCredentials(AzureServicePrincipalCredentials): authentication: str = _DEFAULT_AUTHENTICATION """Authentication method. Driver-native: `ActiveDirectoryServicePrincipal` (default), - `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`. - azure-identity (token injected by dlt): `auto`/`default`, `cli`, `environment`, - `interactive`, `devicecode`, `msi`/`managedidentity`.""" + `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, + `ActiveDirectoryMsi`. azure-identity (token injected by dlt): `ActiveDirectoryDefault` + (alias `default`), `ActiveDirectoryDeviceCode`.""" username: str | None = None """User principal name, used with `ActiveDirectoryPassword` authentication.""" diff --git a/dlt/destinations/impl/mssql/configuration.py b/dlt/destinations/impl/mssql/configuration.py index e7a6b17073..b950b9013e 100644 --- a/dlt/destinations/impl/mssql/configuration.py +++ b/dlt/destinations/impl/mssql/configuration.py @@ -22,57 +22,56 @@ SQL_COPT_SS_ACCESS_TOKEN = 1256 SQL_TOKEN_SCOPE = "https://database.windows.net/.default" -# Authentication methods handled by the ODBC driver itself (passed as `Authentication=` in the DSN). +# Authentication methods the ODBC driver performs natively (passed as `Authentication=` in the +# DSN). Poolable: the driver signs in itself, so no token needs to be reacquired and injected +# per connection. DRIVER_NATIVE_AUTHENTICATION = frozenset( { "ActiveDirectoryServicePrincipal", "ActiveDirectoryPassword", "ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", + "ActiveDirectoryMsi", } ) -# Authentication methods backed by azure-identity. dlt acquires the access token and injects it -# into the connection, so these work cross-platform without relying on ODBC driver support. -AZURE_IDENTITY_AUTHENTICATION = frozenset( +# Authentication methods pyodbc/msodbcsql does not support natively. dlt acquires the access +# token itself with azure-identity and injects it into the connection (`attrs_before`), so these +# work cross-platform but are not poolable. +AZURE_IDENTITY_INJECTION_AUTHENTICATION = frozenset( { - "auto", - "default", - "cli", - "environment", - "interactive", - "devicecode", - "msi", - "managedidentity", + "ActiveDirectoryDefault", + "ActiveDirectoryDeviceCode", } ) +# Thin alias for `ActiveDirectoryDefault`, resolved by `_normalize_authentication`. +_AUTHENTICATION_ALIASES = { + "default": "ActiveDirectoryDefault", +} + + +def _normalize_authentication(authentication: str) -> str: + """Resolve the thin `default` alias to the canonical `ActiveDirectoryDefault` name.""" + return _AUTHENTICATION_ALIASES.get(authentication.lower(), authentication) + def create_token_credential(authentication: str) -> "TokenCredential": - """Create an azure-identity credential for a token-based authentication method.""" + """Create an azure-identity credential for a token-injection authentication method. + + Expects the canonical name (`ActiveDirectoryDefault` or `ActiveDirectoryDeviceCode`), as + already resolved by `_normalize_authentication`. + """ try: - from azure.identity import ( - AzureCliCredential, - DefaultAzureCredential, - DeviceCodeCredential, - EnvironmentCredential, - InteractiveBrowserCredential, - ManagedIdentityCredential, - ) + from azure.identity import DefaultAzureCredential, DeviceCodeCredential except ModuleNotFoundError: raise MissingDependencyException("MsSqlCredentials", [_AZURE_AUTH_EXTRA]) credential_factories = { - "auto": DefaultAzureCredential, - "default": DefaultAzureCredential, - "cli": AzureCliCredential, - "environment": EnvironmentCredential, - "interactive": InteractiveBrowserCredential, - "devicecode": DeviceCodeCredential, - "msi": ManagedIdentityCredential, - "managedidentity": ManagedIdentityCredential, + "ActiveDirectoryDefault": DefaultAzureCredential, + "ActiveDirectoryDeviceCode": DeviceCodeCredential, } - return credential_factories[authentication.lower()]() # type: ignore[no-any-return] + return credential_factories[authentication]() # type: ignore[no-any-return] def uses_token_authentication(credentials: Any) -> bool: @@ -84,7 +83,7 @@ def uses_token_authentication(credentials: Any) -> bool: authentication = credentials.authentication if not authentication: return False - if authentication.lower() in AZURE_IDENTITY_AUTHENTICATION: + if _normalize_authentication(authentication) in AZURE_IDENTITY_INJECTION_AUTHENTICATION: return True # ActiveDirectoryServicePrincipal without a secret cannot authenticate through the ODBC # driver, so dlt falls back to a DefaultAzureCredential token. @@ -96,16 +95,17 @@ def uses_token_authentication(credentials: Any) -> bool: def setup_token_credential(credentials: Any) -> None: - """Create and store the azure-identity credential for token-based authentication.""" + """Create and store the azure-identity credential for token-injection authentication.""" authentication = credentials.authentication or "" - if authentication.lower() in AZURE_IDENTITY_AUTHENTICATION: - credentials._set_default_credentials(create_token_credential(authentication)) + normalized = _normalize_authentication(authentication) + if normalized in AZURE_IDENTITY_INJECTION_AUTHENTICATION: + credentials._set_default_credentials(create_token_credential(normalized)) elif authentication == "ActiveDirectoryServicePrincipal" and not ( credentials.azure_client_id and credentials.azure_client_secret and credentials.azure_tenant_id ): - credentials._set_default_credentials(create_token_credential("default")) + credentials._set_default_credentials(create_token_credential("ActiveDirectoryDefault")) def get_token_credential(credentials: Any) -> "TokenCredential": @@ -116,9 +116,9 @@ def get_token_credential(credentials: Any) -> "TokenCredential": """ if credentials.has_default_credentials(): return credentials.default_credentials() # type: ignore[no-any-return] - authentication = credentials.authentication or "" - if authentication.lower() not in AZURE_IDENTITY_AUTHENTICATION: - authentication = "default" + authentication = _normalize_authentication(credentials.authentication or "") + if authentication not in AZURE_IDENTITY_INJECTION_AUTHENTICATION: + authentication = "ActiveDirectoryDefault" return create_token_credential(authentication) @@ -137,11 +137,14 @@ def validate_authentication(credentials: Any) -> None: authentication = credentials.authentication if not authentication: return # plain SQL login (username/password) + normalized = _normalize_authentication(authentication) if ( - authentication not in DRIVER_NATIVE_AUTHENTICATION - and authentication.lower() not in AZURE_IDENTITY_AUTHENTICATION + normalized not in DRIVER_NATIVE_AUTHENTICATION + and normalized not in AZURE_IDENTITY_INJECTION_AUTHENTICATION ): - supported = sorted(DRIVER_NATIVE_AUTHENTICATION) + sorted(AZURE_IDENTITY_AUTHENTICATION) + supported = sorted(DRIVER_NATIVE_AUTHENTICATION) + sorted( + AZURE_IDENTITY_INJECTION_AUTHENTICATION + ) raise ConfigurationException( f"Unsupported `authentication` method `{authentication}`." f" Supported methods: {', '.join(supported)}." @@ -235,10 +238,11 @@ class MsSqlCredentials(ConnectionStringCredentials, CredentialsWithDefault): authentication: str | None = None """Authentication method. Empty (default) uses plain SQL login (`username`/`password`). - Driver-native: `ActiveDirectoryServicePrincipal`, `ActiveDirectoryPassword`, - `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`. azure-identity (token injected by - dlt): `auto`/`default`, `cli`, `environment`, `interactive`, `devicecode`, - `msi`/`managedidentity`.""" + Driver-native (passed as `Authentication=` in the DSN, poolable): + `ActiveDirectoryServicePrincipal`, `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, + `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`. azure-identity (token acquired and + injected by dlt, not poolable): `ActiveDirectoryDefault` (alias `default`), + `ActiveDirectoryDeviceCode`.""" azure_tenant_id: str | None = None """Entra ID tenant id, used with `ActiveDirectoryServicePrincipal` authentication.""" diff --git a/docs/website/docs/dlt-ecosystem/destinations/fabric.md b/docs/website/docs/dlt-ecosystem/destinations/fabric.md index 1cd53ce5bd..3457ac13bf 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/fabric.md +++ b/docs/website/docs/dlt-ecosystem/destinations/fabric.md @@ -49,6 +49,7 @@ With the **driver-native** methods, the ODBC driver performs the Entra ID sign-i | `ActiveDirectoryPassword` | Entra ID username/password | `username`, `password` | | `ActiveDirectoryIntegrated` | Integrated Windows authentication | None | | `ActiveDirectoryInteractive` | Interactive browser prompt (driver) | None | +| `ActiveDirectoryMsi` | Managed identity (driver) | None | With the **azure-identity** methods, `dlt` acquires an access token with [azure-identity](https://learn.microsoft.com/python/api/overview/azure/identity-readme) and injects @@ -57,15 +58,11 @@ Entra ID modes are unreliable) and need no secret in `secrets.toml`: | `authentication` | azure-identity credential | |---|---| -| `auto` / `default` | `DefaultAzureCredential` | -| `cli` | `AzureCliCredential` (uses `az login`) | -| `environment` | `EnvironmentCredential` | -| `interactive` | `InteractiveBrowserCredential` | -| `devicecode` | `DeviceCodeCredential` | -| `msi` / `managedidentity` | `ManagedIdentityCredential` | +| `ActiveDirectoryDefault` (alias `default`) | `DefaultAzureCredential` (managed identity, environment, Azure CLI, …) | +| `ActiveDirectoryDeviceCode` | `DeviceCodeCredential` | When `authentication` is left at its default but no Service Principal secret is configured, `dlt` -falls back to `DefaultAzureCredential`. +falls back to `ActiveDirectoryDefault` (`DefaultAzureCredential`). ### Create a pipeline @@ -98,13 +95,13 @@ port = 1433 connect_timeout = 30 ``` -azure-identity, e.g. the Azure CLI (`az login`), which needs no secret: +azure-identity, e.g. `DefaultAzureCredential` after `az login`, which needs no secret: ```toml [destination.fabric.credentials] host = ".datawarehouse.fabric.microsoft.com" database = "mydb" -authentication = "cli" +authentication = "default" ``` ## Write disposition diff --git a/docs/website/docs/dlt-ecosystem/destinations/mssql.md b/docs/website/docs/dlt-ecosystem/destinations/mssql.md index 8af58e915d..5c352fa3cc 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/mssql.md +++ b/docs/website/docs/dlt-ecosystem/destinations/mssql.md @@ -109,23 +109,20 @@ They require the `azure-identity` package (installed with `pip install "dlt[az]" | `authentication` | How it authenticates | |---|---| | _(empty, default)_ | SQL login with `username`/`password` | -| `cli` | `AzureCliCredential` (uses `az login`) | -| `auto` / `default` | `DefaultAzureCredential` (managed identity, env, CLI, …) | -| `environment` | `EnvironmentCredential` | -| `interactive` | `InteractiveBrowserCredential` | -| `devicecode` | `DeviceCodeCredential` | -| `msi` / `managedidentity` | `ManagedIdentityCredential` | -| `ActiveDirectoryServicePrincipal` | Service Principal (`azure_tenant_id`, `azure_client_id`, `azure_client_secret`) | +| `ActiveDirectoryServicePrincipal` | Service Principal (`azure_tenant_id`, `azure_client_id`, `azure_client_secret`), handled by the ODBC driver | | `ActiveDirectoryPassword` | Entra ID `username`/`password` (handled by the ODBC driver) | | `ActiveDirectoryIntegrated` | Integrated Windows authentication (handled by the ODBC driver) | | `ActiveDirectoryInteractive` | Interactive prompt (handled by the ODBC driver) | +| `ActiveDirectoryMsi` | Managed identity (handled by the ODBC driver) | +| `ActiveDirectoryDefault` (alias `default`) | `DefaultAzureCredential` (managed identity, environment, Azure CLI, …), token injected by dlt | +| `ActiveDirectoryDeviceCode` | `DeviceCodeCredential`, token injected by dlt | -Passwordless example using the Azure CLI: +Passwordless example using `DefaultAzureCredential` (e.g. after `az login`): ```toml [destination.mssql.credentials] database = "dlt_data" host = "loader.database.windows.net" -authentication = "cli" +authentication = "default" ``` Service Principal example: diff --git a/docs/website/docs/dlt-ecosystem/destinations/synapse.md b/docs/website/docs/dlt-ecosystem/destinations/synapse.md index a0e3ae493e..70dd6841ed 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/synapse.md +++ b/docs/website/docs/dlt-ecosystem/destinations/synapse.md @@ -99,7 +99,8 @@ pipeline = dlt.pipeline( ``` **Microsoft Entra ID authentication** is supported through the same `authentication` credential option as the [mssql destination](./mssql.md#microsoft-entra-id-authentication), including the -passwordless azure-identity methods (`cli`, `default`, `msi`, …) that inject an access token. +passwordless azure-identity methods (`ActiveDirectoryDefault`, `ActiveDirectoryDeviceCode`) that +inject an access token. To use **Active Directory Principal**, you can use the `sqlalchemy.engine.URL.create` method to create the connection URL using your Active Directory Service Principal credentials. First, create the connection string as: ```py diff --git a/tests/load/fabric/test_fabric_configuration.py b/tests/load/fabric/test_fabric_configuration.py index 19897a6dec..c1b3cf6370 100644 --- a/tests/load/fabric/test_fabric_configuration.py +++ b/tests/load/fabric/test_fabric_configuration.py @@ -262,14 +262,9 @@ def test_fabric_authentication_default_is_service_principal() -> None: @pytest.mark.parametrize( "authentication,expected", [ - ("auto", "DefaultAzureCredential"), ("default", "DefaultAzureCredential"), - ("cli", "AzureCliCredential"), - ("environment", "EnvironmentCredential"), - ("interactive", "InteractiveBrowserCredential"), - ("devicecode", "DeviceCodeCredential"), - ("msi", "ManagedIdentityCredential"), - ("managedidentity", "ManagedIdentityCredential"), + ("ActiveDirectoryDefault", "DefaultAzureCredential"), + ("ActiveDirectoryDeviceCode", "DeviceCodeCredential"), ], ) def test_fabric_azure_identity_credential_mapping(authentication: str, expected: str) -> None: @@ -286,6 +281,17 @@ def test_fabric_azure_identity_credential_mapping(authentication: str, expected: assert dsn["LongAsMax"] == "yes" +@pytest.mark.parametrize( + "authentication", + ["auto", "cli", "environment", "interactive", "devicecode", "msi", "managedidentity"], +) +def test_fabric_removed_dlt_custom_alias_raises(authentication: str) -> None: + """The old dlt-custom lowercase aliases were replaced by native ODBC/azure-identity names.""" + creds = _warehouse_credentials(authentication) + with pytest.raises(ConfigurationException): + creds.on_partial() # resolves (host+database present) -> on_resolved -> validate raises + + def test_fabric_service_principal_without_secret_falls_back_to_token() -> None: """Default method without a Service Principal secret injects a DefaultAzureCredential token.""" creds = _warehouse_credentials() @@ -312,7 +318,8 @@ def test_fabric_service_principal_with_secret_is_driver_native() -> None: @pytest.mark.parametrize( - "authentication", ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive"] + "authentication", + ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", "ActiveDirectoryMsi"], ) def test_fabric_driver_native_passthrough(authentication: str) -> None: creds = _warehouse_credentials(authentication) @@ -352,7 +359,7 @@ def test_fabric_unsupported_authentication_raises() -> None: def test_fabric_to_odbc_attrs_before_token_struct() -> None: """The injected token follows the SQL_COPT_SS_ACCESS_TOKEN struct layout.""" - creds = _warehouse_credentials("cli") + creds = _warehouse_credentials("default") creds._set_default_credentials(_FakeTokenCredential()) attrs = creds.to_odbc_attrs_before() @@ -368,11 +375,11 @@ def test_fabric_resolve_configuration_token_authentication() -> None: creds = FabricCredentials() creds.host = "abc.datawarehouse.fabric.microsoft.com" creds.database = "mydb" - creds.authentication = "cli" + creds.authentication = "ActiveDirectoryDeviceCode" resolved = resolve_configuration(creds) assert resolved.is_resolved() assert uses_token_authentication(resolved) is True - assert type(resolved.default_credentials()).__name__ == "AzureCliCredential" + assert type(resolved.default_credentials()).__name__ == "DeviceCodeCredential" assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() diff --git a/tests/load/mssql/test_mssql_configuration.py b/tests/load/mssql/test_mssql_configuration.py index 81aa1411b7..843c7da089 100644 --- a/tests/load/mssql/test_mssql_configuration.py +++ b/tests/load/mssql/test_mssql_configuration.py @@ -263,14 +263,9 @@ def test_mssql_sql_login_dsn_uses_uid_pwd() -> None: @pytest.mark.parametrize( "authentication,expected", [ - ("auto", "DefaultAzureCredential"), ("default", "DefaultAzureCredential"), - ("cli", "AzureCliCredential"), - ("environment", "EnvironmentCredential"), - ("interactive", "InteractiveBrowserCredential"), - ("devicecode", "DeviceCodeCredential"), - ("msi", "ManagedIdentityCredential"), - ("managedidentity", "ManagedIdentityCredential"), + ("ActiveDirectoryDefault", "DefaultAzureCredential"), + ("ActiveDirectoryDeviceCode", "DeviceCodeCredential"), ], ) def test_mssql_azure_identity_credential_mapping(authentication: str, expected: str) -> None: @@ -285,6 +280,17 @@ def test_mssql_azure_identity_credential_mapping(authentication: str, expected: assert "PWD" not in dsn +@pytest.mark.parametrize( + "authentication", + ["auto", "cli", "environment", "interactive", "devicecode", "msi", "managedidentity"], +) +def test_mssql_removed_dlt_custom_alias_raises(authentication: str) -> None: + """The old dlt-custom lowercase aliases were replaced by native ODBC/azure-identity names.""" + creds = _mssql_credentials(authentication) + with pytest.raises(ConfigurationException): + validate_authentication(creds) + + def test_mssql_service_principal_driver_native() -> None: creds = _mssql_credentials( "ActiveDirectoryServicePrincipal", @@ -312,7 +318,8 @@ def test_mssql_service_principal_without_secret_falls_back_to_token() -> None: @pytest.mark.parametrize( - "authentication", ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive"] + "authentication", + ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", "ActiveDirectoryMsi"], ) def test_mssql_driver_native_passthrough(authentication: str) -> None: creds = _mssql_credentials(authentication) @@ -350,7 +357,7 @@ def test_mssql_unsupported_authentication_raises() -> None: def test_mssql_to_odbc_attrs_before_token_struct() -> None: - creds = _mssql_credentials("cli") + creds = _mssql_credentials("default") creds._set_default_credentials(_FakeTokenCredential()) attrs = creds.to_odbc_attrs_before() @@ -366,11 +373,11 @@ def test_mssql_resolve_configuration_token_authentication() -> None: creds.host = "sql.example.com" creds.database = "test_db" creds.driver = "ODBC Driver 18 for SQL Server" - creds.authentication = "cli" + creds.authentication = "ActiveDirectoryDeviceCode" resolved = resolve_configuration(creds) assert resolved.is_resolved() assert uses_token_authentication(resolved) is True - assert type(resolved.default_credentials()).__name__ == "AzureCliCredential" + assert type(resolved.default_credentials()).__name__ == "DeviceCodeCredential" assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() From 5c0a4a3c6f237ffd39b2b5e96499a82c83920061 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Tue, 30 Jun 2026 19:17:25 +0200 Subject: [PATCH 3/4] feat(mssql): migrate mssql/synapse/fabric to the mssql-python driver Replace pyodbc with Microsoft's mssql-python driver for the mssql, synapse and fabric destinations. mssql-python bundles the SQL Server client libraries, so no separate ODBC driver installation is required, and it works natively across platforms (including arm64 macOS). - MsSqlClient (formerly PyOdbcMsSqlClient, kept as an alias) connects via mssql_python.connect with attrs_before token injection and autocommit. - datetimeoffset is returned natively as a tz-aware datetime, so the pyodbc output-converter workaround is removed. - Database exceptions are mapped from mssql-python's typed exception hierarchy. - The ODBC driver-selection machinery (pyodbc.drivers(), SUPPORTED_DRIVERS, driver validation) is removed and no DRIVER is emitted in the connection string. The `driver` credential option is kept but deprecated and ignored for backwards compatibility. - The mssql/synapse/fabric extras now depend on mssql-python instead of pyodbc. Adds a Docker-based integration test that loads data end-to-end through mssql-python, and updates the docs and lockfile. --- .github/workflows/test_destinations_local.yml | 23 ++- .../workflows/test_destinations_remote.yml | 3 - dlt/destinations/impl/fabric/configuration.py | 18 +-- dlt/destinations/impl/mssql/configuration.py | 50 ++----- dlt/destinations/impl/mssql/sql_client.py | 121 ++++++--------- .../impl/synapse/configuration.py | 13 +- .../docs/dlt-ecosystem/destinations/fabric.md | 37 +---- .../docs/dlt-ecosystem/destinations/mssql.md | 34 +---- .../dlt-ecosystem/destinations/synapse.md | 14 +- pyproject.toml | 7 +- tests/.dlt/dev.secrets.toml | 3 + .../load/fabric/test_fabric_configuration.py | 23 ++- tests/load/mssql/test_mssql_configuration.py | 90 +++-------- tests/load/mssql/test_mssql_docker_auth.py | 61 ++++++-- .../synapse/test_synapse_configuration.py | 29 ++-- uv.lock | 141 +++++++----------- 16 files changed, 268 insertions(+), 399 deletions(-) diff --git a/.github/workflows/test_destinations_local.yml b/.github/workflows/test_destinations_local.yml index 93fa595933..d89909e5e1 100644 --- a/.github/workflows/test_destinations_local.yml +++ b/.github/workflows/test_destinations_local.yml @@ -58,6 +58,14 @@ jobs: # NOTE: we only run non-staging tests, as staging tests require credentials for s3 and azure excluded_destination_configurations: "[\"clickhouse-parquet-staging-s3-authorization\", \"clickhouse-parquet-staging-az-authorization\", \"clickhouse-jsonl-staging-az-authorization\", \"clickhouse-jsonl-staging-s3-authorization\"]" + # MSSQL (local SQL Server in Docker, no cloud credentials required). + # The mssql-python driver bundles its client libraries, so no ODBC driver install is needed. + - display_name: mssql + destinations: "[\"mssql\"]" + filesystem_drivers: "[\"memory\"]" + extras: "--extra mssql --extra parquet" + needs_mssql: true + # Dremio - display_name: dremio destinations: "[\"dremio\"]" @@ -167,6 +175,15 @@ jobs: name: Start ClickHouse OSS if: ${{ matrix.needs_clickhouse }} + - name: Start SQL Server and create database + if: ${{ matrix.needs_mssql }} + run: | + docker compose -f "tests/load/mssql/docker-compose.yml" up -d + echo "Waiting for SQL Server to accept connections..." + timeout 90s bash -c 'until docker exec dlt_test_mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "Strong!Passw0rd" -C -Q "SELECT 1" >/dev/null 2>&1; do sleep 2; done' + docker exec dlt_test_mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "Strong!Passw0rd" -C -Q "IF DB_ID('dlt_data') IS NULL CREATE DATABASE dlt_data" + echo "SQL Server is up and dlt_data is ready" + # # Setup python and run tests # @@ -215,4 +232,8 @@ jobs: - name: Stop ClickHouse OSS if: always() - run: docker compose -f "tests/load/clickhouse/docker-compose.yml" down -v \ No newline at end of file + run: docker compose -f "tests/load/clickhouse/docker-compose.yml" down -v + + - name: Stop SQL Server + if: always() + run: docker compose -f "tests/load/mssql/docker-compose.yml" down -v \ No newline at end of file diff --git a/.github/workflows/test_destinations_remote.yml b/.github/workflows/test_destinations_remote.yml index 56165c7704..c1cece0cac 100644 --- a/.github/workflows/test_destinations_remote.yml +++ b/.github/workflows/test_destinations_remote.yml @@ -148,7 +148,6 @@ jobs: destinations: "[\"mssql\"]" filesystem_drivers: "[\"memory\"]" extras: "--extra mssql --extra s3 --extra gs --extra az --extra parquet --group adbc" - pre_install_commands: "sudo apt-get update && sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18" post_install_commands: "uv run dbc install mssql" always_run_all_tests: true @@ -157,14 +156,12 @@ jobs: destinations: "[\"synapse\"]" filesystem_drivers: "[\"memory\"]" extras: "--extra synapse --extra parquet" - pre_install_commands: "sudo apt-get update && sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18" # Fabric - display_name: fabric destinations: "[\"fabric\"]" filesystem_drivers: "[\"memory\"]" extras: "--extra fabric" - pre_install_commands: "sudo apt-get update && sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18" post_secrets_commands: "uv run python tests/load/fabric/refresh_spn_fabric_token.py" # Postgres and Redshift (used to be test_destinations.yml) diff --git a/dlt/destinations/impl/fabric/configuration.py b/dlt/destinations/impl/fabric/configuration.py index c74bd993ce..3377f1af5a 100644 --- a/dlt/destinations/impl/fabric/configuration.py +++ b/dlt/destinations/impl/fabric/configuration.py @@ -37,9 +37,6 @@ class FabricCredentials(AzureServicePrincipalCredentials): Inherits from AzureServicePrincipalCredentials for the Service Principal fields. """ - drivername: str = "mssql+pyodbc" - """SQLAlchemy driver name for SQL Server/Fabric.""" - host: str = None """Fabric Warehouse host (e.g., abc12345-6789-def0-1234-56789abcdef0.datawarehouse.fabric.microsoft.com)""" @@ -85,12 +82,15 @@ def on_resolved(self) -> None: validate_authentication(self) def get_odbc_dsn_dict(self) -> Dict[str, Any]: - """Build ODBC DSN dictionary with Fabric-specific settings.""" + """Build ODBC DSN dictionary with Fabric-specific settings. + + mssql-python bundles its own driver, so no DRIVER key is emitted. + """ params: dict[str, Any] = { - "DRIVER": "{ODBC Driver 18 for SQL Server}", "SERVER": f"{self.host},{self.port}", "DATABASE": self.database, - "LongAsMax": "yes", # Required for UTF-8 collation support + # The mssql-python driver handles long/max types (e.g. varchar(max)) natively, + # so no LongAsMax keyword is needed. "Encrypt": "yes", "TrustServerCertificate": "no", } @@ -98,11 +98,11 @@ def get_odbc_dsn_dict(self) -> Dict[str, Any]: return params def to_odbc_attrs_before(self) -> dict[int, bytes] | None: - """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" return build_token_attrs_before(self) def to_odbc_dsn(self) -> str: - """Build ODBC connection string for pyodbc.""" + """Build the ODBC connection string.""" params = self.get_odbc_dsn_dict() return ";".join(f"{k}={v}" for k, v in params.items()) @@ -192,7 +192,7 @@ class FabricClientConfiguration(DestinationClientDwhWithStagingConfiguration): - Latin1_General_100_BIN2_UTF8 (default, case-sensitive) - Latin1_General_100_CI_AS_KS_WS_SC_UTF8 (case-insensitive) - Both have UTF-8 encoding. LongAsMax=yes is automatically configured. + Both have UTF-8 encoding. Long/max types are handled natively by the driver. """ def physical_location(self) -> str: diff --git a/dlt/destinations/impl/mssql/configuration.py b/dlt/destinations/impl/mssql/configuration.py index b950b9013e..a624dab588 100644 --- a/dlt/destinations/impl/mssql/configuration.py +++ b/dlt/destinations/impl/mssql/configuration.py @@ -7,7 +7,7 @@ from dlt.common.configuration.exceptions import ConfigurationException from dlt.common.configuration.specs import ConnectionStringCredentials, CredentialsWithDefault from dlt.common.typing import TSecretStrValue -from dlt.common.exceptions import MissingDependencyException, SystemConfigurationException +from dlt.common.exceptions import MissingDependencyException from dlt.common.destination.client import DestinationClientDwhWithStagingConfiguration from dlt.common.utils import digest128 @@ -17,7 +17,7 @@ _AZURE_AUTH_EXTRA = f"{version.DLT_PKG_NAME}[az]" -# pyodbc connection attribute used to inject a pre-acquired Entra ID access token. +# ODBC connection attribute used to inject a pre-acquired Entra ID access token. # https://learn.microsoft.com/sql/connect/odbc/using-azure-active-directory#authenticating-with-an-access-token SQL_COPT_SS_ACCESS_TOKEN = 1256 SQL_TOKEN_SCOPE = "https://database.windows.net/.default" @@ -123,7 +123,7 @@ def get_token_credential(credentials: Any) -> "TokenCredential": def build_token_attrs_before(credentials: Any) -> dict[int, bytes] | None: - """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" if not uses_token_authentication(credentials): return None token = get_token_credential(credentials).get_token(SQL_TOKEN_SCOPE).token @@ -234,7 +234,8 @@ class MsSqlCredentials(ConnectionStringCredentials, CredentialsWithDefault): host: str = None port: int = 1433 connect_timeout: int = 30 - driver: str = None + driver: Optional[str] = None + """Deprecated and ignored: mssql-python bundles its driver, so no ODBC driver name is needed.""" authentication: str | None = None """Authentication method. Empty (default) uses plain SQL login (`username`/`password`). @@ -255,11 +256,6 @@ class MsSqlCredentials(ConnectionStringCredentials, CredentialsWithDefault): __config_gen_annotations__: ClassVar[List[str]] = ["port", "connect_timeout"] - SUPPORTED_DRIVERS: ClassVar[List[str]] = [ - "ODBC Driver 18 for SQL Server", - "ODBC Driver 17 for SQL Server", - ] - def parse_native_representation(self, native_value: Any) -> None: # TODO: Support ODBC connection string or sqlalchemy URL super().parse_native_representation(native_value) @@ -270,11 +266,6 @@ def parse_native_representation(self, native_value: Any) -> None: def on_resolved(self) -> None: validate_authentication(self) - if self.driver not in self.SUPPORTED_DRIVERS: - raise SystemConfigurationException( - f"The specified driver `{self.driver}` is not supported." - f" Choose one of the supported drivers: {', '.join(self.SUPPORTED_DRIVERS)}." - ) self.database = self.database.lower() def get_query(self) -> Dict[str, Any]: @@ -283,7 +274,6 @@ def get_query(self) -> Dict[str, Any]: return query def on_partial(self) -> None: - self.driver = self._get_driver() setup_token_credential(self) if self.authentication: # Entra ID methods (token or driver-native) supply their own credentials and do not @@ -294,32 +284,22 @@ def on_partial(self) -> None: # Plain SQL login needs username/password. self.resolve() - def _get_driver(self) -> str: - if self.driver: - return self.driver - - # Pick a default driver if available - import pyodbc - - available_drivers = pyodbc.drivers() - for d in self.SUPPORTED_DRIVERS: - if d in available_drivers: - return d - docs_url = "https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16" - raise SystemConfigurationException( - f"No supported ODBC driver found for MS SQL Server. See {docs_url} for information on" - f" how to install the `{self.SUPPORTED_DRIVERS[0]}` on your platform." - ) - def get_odbc_dsn_dict(self) -> Dict[str, Any]: + # mssql-python bundles its own driver, so no DRIVER key is emitted. params: dict[str, Any] = { - "DRIVER": self.driver, "SERVER": f"{self.host},{self.port}", "DATABASE": self.database, } apply_authentication_to_dsn(self, params) if self.query is not None: - params.update({k.upper(): v for k, v in self.query.items()}) + # mssql-python's connection-string parser rejects unknown keywords. `connect_timeout` + # is passed separately via the connect() `timeout=` parameter, and `longasmax` is + # unnecessary since the driver handles long/max types natively, so neither belongs + # in the DSN. + skip_keys = {"driver", "connect_timeout", "longasmax"} + params.update( + {k.upper(): v for k, v in self.query.items() if k.lower() not in skip_keys} + ) return params def to_odbc_dsn(self) -> str: @@ -327,7 +307,7 @@ def to_odbc_dsn(self) -> str: return build_odbc_dsn(params) def to_odbc_attrs_before(self) -> dict[int, bytes] | None: - """Return pyodbc `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" return build_token_attrs_before(self) diff --git a/dlt/destinations/impl/mssql/sql_client.py b/dlt/destinations/impl/mssql/sql_client.py index ab5d9e6fdd..ad654d0ff0 100644 --- a/dlt/destinations/impl/mssql/sql_client.py +++ b/dlt/destinations/impl/mssql/sql_client.py @@ -1,9 +1,6 @@ -import struct -from datetime import datetime, timedelta, timezone # noqa: I251 - from dlt.common.destination import DestinationCapabilitiesContext -import pyodbc +import mssql_python from contextlib import contextmanager from typing import Any, AnyStr, ClassVar, Iterator, Optional, Sequence, Tuple @@ -25,23 +22,8 @@ from dlt.common.destination.dataset import DBApiCursor -def handle_datetimeoffset(dto_value: bytes) -> datetime: - # ref: https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794 - tup = struct.unpack("<6hI2h", dto_value) # e.g., (2017, 3, 16, 10, 35, 18, 500000000, -6, 0) - return datetime( - tup[0], - tup[1], - tup[2], - tup[3], - tup[4], - tup[5], - tup[6] // 1000, - timezone(timedelta(hours=tup[7], minutes=tup[8])), - ) - - -class PyOdbcMsSqlClient(SqlClientBase[pyodbc.Connection], DBTransaction): - dbapi: ClassVar[DBApi] = pyodbc +class MsSqlClient(SqlClientBase[mssql_python.Connection], DBTransaction): + dbapi: ClassVar[DBApi] = mssql_python def __init__( self, @@ -51,26 +33,18 @@ def __init__( capabilities: DestinationCapabilitiesContext, ) -> None: super().__init__(credentials.database, dataset_name, staging_dataset_name, capabilities) - self._conn: pyodbc.Connection = None + self._conn: mssql_python.Connection = None self.credentials = credentials - def open_connection(self) -> pyodbc.Connection: - # For Entra ID token authentication, inject a fresh access token via attrs_before. - attrs_before = self.credentials.to_odbc_attrs_before() - if attrs_before: - self._conn = pyodbc.connect( - self.credentials.to_odbc_dsn(), - timeout=self.credentials.connect_timeout, - attrs_before=attrs_before, - ) - else: - self._conn = pyodbc.connect( - self.credentials.to_odbc_dsn(), - timeout=self.credentials.connect_timeout, - ) - # https://github.com/mkleehammer/pyodbc/wiki/Using-an-Output-Converter-function - self._conn.add_output_converter(-155, handle_datetimeoffset) - self._conn.autocommit = True + def open_connection(self) -> mssql_python.Connection: + # mssql-python bundles its own driver, so the connection string carries no DRIVER. For + # Entra ID token authentication a fresh access token is injected via attrs_before. + self._conn = mssql_python.connect( + self.credentials.to_odbc_dsn(), + autocommit=True, + attrs_before=self.credentials.to_odbc_attrs_before(), # type: ignore[arg-type] + timeout=self.credentials.connect_timeout, + ) return self._conn @raise_open_connection_error @@ -96,20 +70,14 @@ def commit_transaction(self) -> None: @raise_database_error def rollback_transaction(self) -> None: + # mssql-python treats a rollback without an active transaction as a no-op. try: self._conn.rollback() - except pyodbc.ProgrammingError as ex: - if ( - ex.args[0] == "42000" and "(111214)" in ex.args[1] - ): # "no corresponding transaction found" - pass # there was nothing to rollback, we silently ignore the error - else: - raise finally: self._conn.autocommit = True @property - def native_connection(self) -> pyodbc.Connection: + def native_connection(self) -> mssql_python.Connection: return self._conn def drop_dataset(self) -> None: @@ -156,36 +124,38 @@ def execute_sql( @raise_database_error def execute_query(self, query: AnyStr, *args: Any, **kwargs: Any) -> Iterator[DBApiCursor]: assert isinstance(query, str) - if kwargs: - raise NotImplementedError("pyodbc does not support named parameters in queries") if args: + # dlt emits %s positional placeholders; mssql-python expects qmark (?) # TODO: this is bad. See duckdb & athena also query = query.replace("%s", "?") # NOTE: do not convert it into context manager. it does not close the cursor! curr = self._conn.cursor() try: - # unpack because empty tuple gets interpreted as a single argument - # https://github.com/mkleehammer/pyodbc/wiki/Features-beyond-the-DB-API#passing-parameters - curr.execute(query, *args) + if kwargs: + # mssql-python's paramstyle is pyformat: pass named parameters (%(name)s) through + curr.execute(query, kwargs) + else: + # unpack because empty tuple gets interpreted as a single argument + curr.execute(query, *args) # NOTE: firsts recordset is wrapped in a cursor - yield DBApiCursorImpl(curr) + yield DBApiCursorImpl(curr) # type: ignore[arg-type] # clear all pending result sets try: while curr.nextset(): pass - except pyodbc.Error: + except mssql_python.Error: pass - except pyodbc.Error: + except mssql_python.Error: # clear all pending result sets try: while curr.nextset(): pass - except pyodbc.Error: + except mssql_python.Error: pass # immediately rollback transaction try: self._conn.rollback() - except pyodbc.Error: + except mssql_python.Error: pass raise finally: @@ -197,27 +167,34 @@ def execute_query(self, query: AnyStr, *args: Any, **kwargs: Any) -> Iterator[DB @classmethod def _make_database_exception(cls, ex: Exception) -> Exception: - if isinstance(ex, pyodbc.ProgrammingError): - if ex.args[0] == "42S02": + # mssql-python maps the SQLSTATE to a stable `driver_error` label, which we classify on + # (the ddbc_error message is server/locale dependent, the label is not). + driver_error = getattr(ex, "driver_error", "") + if isinstance(ex, mssql_python.ProgrammingError): + if driver_error == "Base table or view not found": # SQLSTATE 42S02 return DatabaseUndefinedRelation(ex) - # certain pyodbc exceptions do not have second argument - if len(ex.args) > 1: - if ex.args[1] == "HY000": - return DatabaseTransientException(ex) - elif ex.args[0] == "42000": - if "(15151)" in ex.args[1]: - return DatabaseUndefinedRelation(ex) - return DatabaseTransientException(ex) - elif isinstance(ex, pyodbc.OperationalError): - return DatabaseTransientException(ex) - elif isinstance(ex, pyodbc.Error): - if ex.args[0] == "07002": # incorrect number of arguments supplied + if driver_error == "Syntax error or access violation": # SQLSTATE 42000 + # error 15151 ("Cannot drop the ... because it does not exist") shares this + # SQLSTATE with real syntax errors; mssql-python drops the error number from the + # message, so match on the text as well. + msg = str(ex) + if "(15151)" in msg or "does not exist" in msg: + return DatabaseUndefinedRelation(ex) + return DatabaseTransientException(ex) + if driver_error == "COUNT field incorrect": # SQLSTATE 07002, wrong parameter count return DatabaseTransientException(ex) + return DatabaseTerminalException(ex) + if isinstance(ex, mssql_python.OperationalError): + return DatabaseTransientException(ex) return DatabaseTerminalException(ex) @staticmethod def is_dbapi_exception(ex: Exception) -> bool: - return isinstance(ex, pyodbc.Error) + return isinstance(ex, mssql_python.Error) def _limit_clause_sql(self, limit: int) -> Tuple[str, str]: return f"TOP ({limit})", "" + + +# Backwards-compatible alias: this client now uses the mssql-python driver instead of pyodbc. +PyOdbcMsSqlClient = MsSqlClient diff --git a/dlt/destinations/impl/synapse/configuration.py b/dlt/destinations/impl/synapse/configuration.py index 257f89b1e3..7ae4592e2f 100644 --- a/dlt/destinations/impl/synapse/configuration.py +++ b/dlt/destinations/impl/synapse/configuration.py @@ -1,6 +1,6 @@ import dataclasses from dlt import version -from typing import Final, Any, List, Dict, Optional, ClassVar +from typing import Final, List, Optional, ClassVar from dlt.common.configuration import configspec from dlt.common.destination.client import DestinationClientConfiguration @@ -17,17 +17,6 @@ class SynapseCredentials(MsSqlCredentials): drivername: Final[str] = dataclasses.field(default="synapse", init=False, repr=False, compare=False) # type: ignore - # LongAsMax keyword got introduced in ODBC Driver 18 for SQL Server. - SUPPORTED_DRIVERS: ClassVar[List[str]] = ["ODBC Driver 18 for SQL Server"] - - def get_odbc_dsn_dict(self) -> Dict[str, Any]: - params = super().get_odbc_dsn_dict() - # Long types (text, ntext, image) are not supported on Synapse. - # Convert to max types using LongAsMax keyword. - # https://stackoverflow.com/a/57926224 - params["LONGASMAX"] = "yes" - return params - @configspec class SynapseClientConfiguration(MsSqlClientConfiguration): diff --git a/docs/website/docs/dlt-ecosystem/destinations/fabric.md b/docs/website/docs/dlt-ecosystem/destinations/fabric.md index 3457ac13bf..ffda58b133 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/fabric.md +++ b/docs/website/docs/dlt-ecosystem/destinations/fabric.md @@ -20,14 +20,9 @@ This will install `dlt` with the `mssql` extra, which contains all the dependenc ### Prerequisites -The _Microsoft ODBC Driver for SQL Server_ must be installed to use this destination. -This cannot be included with `dlt`'s Python dependencies, so you must install it separately on your system. You can find the official installation instructions [here](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16). - -Supported driver versions: -* `ODBC Driver 18 for SQL Server` (recommended) -* `ODBC Driver 17 for SQL Server` - -You can also [configure the driver name](#additional-destination-options) explicitly. +This destination uses the [mssql-python](https://github.com/microsoft/mssql-python) driver, which is +installed automatically with `dlt[fabric]` and bundles the SQL Server client libraries. No separate +ODBC driver installation is required. ### Authentication @@ -198,7 +193,7 @@ Fabric does not support native JSON columns. JSON objects are stored as `varchar ## Collation Support -Fabric Warehouse supports UTF-8 collations. The destination automatically configures `LongAsMax=yes` which is required for UTF-8 collations to work properly. +Fabric Warehouse supports UTF-8 collations. Long/max types (e.g. `varchar(max)`) are handled natively by the mssql-python driver, so no extra configuration is needed for UTF-8 collations to work properly. **Default collation**: `Latin1_General_100_BIN2_UTF8` (case-sensitive, UTF-8) @@ -226,11 +221,8 @@ The **fabric** destination **does not** create UNIQUE indexes by default on colu create_indexes=true ``` -You can explicitly set the ODBC driver name: -```toml -[destination.fabric.credentials] -driver="ODBC Driver 18 for SQL Server" -``` +The `driver` credential option is deprecated and ignored: mssql-python bundles its own driver, so +no ODBC driver name needs to be configured. ## Differences from MSSQL Destination @@ -238,7 +230,7 @@ While Fabric Warehouse is based on SQL Server, there are key differences: 1. **Authentication**: Fabric uses Entra ID; in addition to Service Principal, `dlt` supports several azure-identity methods (see [Authentication](#authentication)) 2. **Type System**: Uses `varchar` and `datetime2` instead of `nvarchar` and `datetimeoffset` -3. **Collation**: Optimized for UTF-8 collations with automatic `LongAsMax` configuration +3. **Collation**: Optimized for UTF-8 collations, with long/max types handled natively by the driver 4. **SQL Dialect**: Uses `fabric` SQLglot dialect for proper SQL generation ### dbt support @@ -246,18 +238,6 @@ Integration with [dbt](../transformations/dbt/dbt.md) is supported via [dbt-fabr ## Troubleshooting -### ODBC Driver Not Found - -If you see "No supported ODBC driver found", install the Microsoft ODBC Driver 18 for SQL Server: - -```sh -# Ubuntu/Debian -curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - -curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list -sudo apt-get update -sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 -``` - ### Authentication Failures Ensure your Service Principal has: @@ -269,8 +249,7 @@ Ensure your Service Principal has: If you experience character encoding issues: 1. Verify your warehouse uses a UTF-8 collation -2. Check that `LongAsMax=yes` is in the connection (automatically added by this destination) -3. Consider using the case-insensitive UTF-8 collation if needed +2. Consider using the case-insensitive UTF-8 collation if needed ## Additional Resources diff --git a/docs/website/docs/dlt-ecosystem/destinations/mssql.md b/docs/website/docs/dlt-ecosystem/destinations/mssql.md index 5c352fa3cc..f096feb9af 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/mssql.md +++ b/docs/website/docs/dlt-ecosystem/destinations/mssql.md @@ -18,14 +18,9 @@ pip install "dlt[mssql]" ### Prerequisites -The _Microsoft ODBC Driver for SQL Server_ must be installed to use this destination. -This cannot be included with `dlt`'s Python dependencies, so you must install it separately on your system. You can find the official installation instructions [here](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16). - -Supported driver versions: -* `ODBC Driver 18 for SQL Server` -* `ODBC Driver 17 for SQL Server` - -You can also [configure the driver name](#additional-destination-options) explicitly. +This destination uses the [mssql-python](https://github.com/microsoft/mssql-python) driver, which is +installed automatically with `dlt[mssql]` and bundles the SQL Server client libraries. No separate +ODBC driver installation is required. ### Create a pipeline @@ -60,14 +55,12 @@ connect_timeout = 15 TrustServerCertificate="yes" # require SSL connection Encrypt="yes" -# send large string as VARCHAR, not legacy TEXT -LongAsMax="yes" ``` You can also pass a SQLAlchemy-like database connection: ```toml # Keep it at the top of your TOML file, before any section starts -destination.mssql.credentials="mssql://loader:@loader.database.windows.net/dlt_data?TrustServerCertificate=yes&Encrypt=yes&LongAsMax=yes" +destination.mssql.credentials="mssql://loader:@loader.database.windows.net/dlt_data?TrustServerCertificate=yes&Encrypt=yes" ``` You can place any ODBC-specific settings into the query string or **destination.mssql.credentials.query** TOML table as in the example above. @@ -92,10 +85,7 @@ destination.mssql.credentials="mssql://loader:loader@localhost/dlt_data?encrypt= destination.mssql.credentials="mssql://loader:loader@localhost/dlt_data?TrustServerCertificate=yes" ``` -**To use long strings (>8k) and avoid collation errors**: -```toml -destination.mssql.credentials="mssql://loader:loader@localhost/dlt_data?LongAsMax=yes" -``` +Long strings (>8k) are handled automatically by the driver, no extra configuration is needed. ### Microsoft Entra ID authentication @@ -240,18 +230,8 @@ The **mssql** destination **does not** create UNIQUE indexes by default on colum create_indexes=true ``` -You can explicitly set the ODBC driver name: -```toml -[destination.mssql.credentials] -driver="ODBC Driver 18 for SQL Server" -``` - -When using a SQLAlchemy connection string, replace spaces with `+`: - -```toml -# Keep it at the top of your TOML file, before any section starts -destination.mssql.credentials="mssql://loader:@loader.database.windows.net/dlt_data?driver=ODBC+Driver+18+for+SQL+Server" -``` +The `driver` credential option is deprecated and ignored: mssql-python bundles its own driver, so +no ODBC driver name needs to be configured. ### dbt support This destination [integrates with dbt](../transformations/dbt/dbt.md) via [dbt-sqlserver](https://github.com/dbt-msft/dbt-sqlserver). diff --git a/docs/website/docs/dlt-ecosystem/destinations/synapse.md b/docs/website/docs/dlt-ecosystem/destinations/synapse.md index 70dd6841ed..3e0136c51f 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/synapse.md +++ b/docs/website/docs/dlt-ecosystem/destinations/synapse.md @@ -18,15 +18,13 @@ pip install "dlt[synapse]" ### Prerequisites -* **Microsoft ODBC Driver for SQL Server** +* **SQL Server driver** - The _Microsoft ODBC Driver for SQL Server_ must be installed to use this destination. - This cannot be included with `dlt`'s Python dependencies, so you must install it separately on your system. You can find the official installation instructions [here](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16). - - Supported driver versions: - * `ODBC Driver 18 for SQL Server` - - > 💡 Older driver versions do not work properly because they do not support the `LongAsMax` keyword that was [introduced](https://learn.microsoft.com/en-us/sql/connect/odbc/windows/features-of-the-microsoft-odbc-driver-for-sql-server-on-windows?view=sql-server-ver15#microsoft-odbc-driver-180-for-sql-server-on-windows) in `ODBC Driver 18 for SQL Server`. Synapse does not support the legacy ["long data types"](https://learn.microsoft.com/en-us/sql/t-sql/data-types/ntext-text-and-image-transact-sql), and requires "max data types" instead. `dlt` uses the `LongAsMax` keyword to automatically do the conversion. + This destination uses the [mssql-python](https://github.com/microsoft/mssql-python) driver, which + is installed automatically with `dlt[synapse]` and bundles the SQL Server client libraries. No + separate ODBC driver installation is required. The driver handles legacy "long data types" + (which Synapse does not support) as "max data types" natively, so no `LongAsMax` keyword + is needed. * **Azure Synapse Workspace and dedicated SQL pool** You need an Azure Synapse workspace with a dedicated SQL pool to load data into. If you do not have one yet, you can use this [quickstart](https://learn.microsoft.com/en-us/azure/synapse-analytics/quickstart-create-sql-pool-studio). diff --git a/pyproject.toml b/pyproject.toml index c6863b4ad4..5820c10e4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,15 +147,15 @@ weaviate = [ "weaviate-client>=4.0.0,<5.0.0" ] mssql = [ - "pyodbc>=4.0.39" + "mssql-python>=1.4.0,!=1.7.0,!=1.7.1" ] synapse = [ - "pyodbc>=4.0.39", + "mssql-python>=1.4.0,!=1.7.0,!=1.7.1", "adlfs>=2024.7.0", "pyarrow>=16.0.0", ] fabric = [ - "pyodbc>=4.0.39", + "mssql-python>=1.4.0,!=1.7.0,!=1.7.1", "adlfs>=2024.7.0", "pyarrow>=16.0.0", ] @@ -487,6 +487,7 @@ module = [ # TODO: Stubs are included but seem to not be installed correctly with pyodbc 4.x.x # Might be fixed in pyodbc 5 "pyodbc.*", + "mssql_python.*", "prometheus_client.*", "weaviate.*", "psycopg2cffi.*", diff --git a/tests/.dlt/dev.secrets.toml b/tests/.dlt/dev.secrets.toml index a690b7b30f..671f104d81 100644 --- a/tests/.dlt/dev.secrets.toml +++ b/tests/.dlt/dev.secrets.toml @@ -29,6 +29,9 @@ module_config = "{\"text2vec-contextionary\": {\"vectorizeClassName\": false, \" [destination.postgres] credentials = "postgresql://loader:loader@localhost:5432/dlt_data" +[destination.mssql] +credentials = "mssql://sa:Strong!Passw0rd@localhost:1433/dlt_data?TrustServerCertificate=yes&Encrypt=no" + # NOTE: specified directly in code # [destination.qdrant.credentials] # location = "http://localhost:6333" diff --git a/tests/load/fabric/test_fabric_configuration.py b/tests/load/fabric/test_fabric_configuration.py index c1b3cf6370..cd0ec26e5a 100644 --- a/tests/load/fabric/test_fabric_configuration.py +++ b/tests/load/fabric/test_fabric_configuration.py @@ -66,10 +66,11 @@ def test_fabric_credentials_odbc_dsn() -> None: # Verify Fabric-specific parameters are added assert dsn_dict["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" - assert dsn_dict["LongAsMax"] == "yes" + assert "LongAsMax" not in dsn_dict assert dsn_dict["UID"] == "test-client-id@test-tenant-id" assert dsn_dict["PWD"] == "test-client-secret" - assert dsn_dict["DRIVER"] == "{ODBC Driver 18 for SQL Server}" + # mssql-python bundles its own driver, so the DSN carries no DRIVER key + assert "DRIVER" not in dsn_dict assert ( dsn_dict["SERVER"] == "abc12345-6789-def0-1234-56789abcdef0.datawarehouse.fabric.microsoft.com,1433" @@ -147,13 +148,6 @@ def test_fabric_type_mapper() -> None: assert "datetimeoffset" not in result.lower() -def test_fabric_credentials_drivername() -> None: - """Test that Fabric credentials use mssql+pyodbc drivername""" - creds = FabricCredentials() - # FabricCredentials uses mssql+pyodbc for SQLAlchemy compatibility - assert creds.drivername == "mssql+pyodbc" - - def test_fabric_credentials_missing_service_principal() -> None: """Test that Service Principal fields can trigger default credentials fallback""" creds = FabricCredentials() @@ -198,8 +192,9 @@ def test_fabric_credentials_no_driver_validation() -> None: assert creds.database == "test_db" -def test_fabric_credentials_longasmax_always_yes() -> None: - """Test that LONGASMAX is always set to 'yes' for UTF-8 support""" +def test_fabric_credentials_longasmax_absent() -> None: + """Test that LongAsMax is never emitted: mssql-python handles long/max types natively + and rejects unknown DSN keywords.""" creds = FabricCredentials() creds.host = "test.datawarehouse.fabric.microsoft.com" creds.database = "testdb" @@ -207,9 +202,9 @@ def test_fabric_credentials_longasmax_always_yes() -> None: creds.azure_client_id = "test-client" creds.azure_client_secret = "test-secret" - # Get ODBC DSN and verify LONGASMAX is set to yes + # Get ODBC DSN and verify LongAsMax is absent dsn_dict = creds.get_odbc_dsn_dict() - assert dsn_dict["LongAsMax"] == "yes" + assert "LongAsMax" not in dsn_dict def test_fabric_credentials_authentication_method() -> None: @@ -278,7 +273,7 @@ def test_fabric_azure_identity_credential_mapping(authentication: str, expected: assert "AUTHENTICATION" not in dsn assert "UID" not in dsn assert "PWD" not in dsn - assert dsn["LongAsMax"] == "yes" + assert "LongAsMax" not in dsn @pytest.mark.parametrize( diff --git a/tests/load/mssql/test_mssql_configuration.py b/tests/load/mssql/test_mssql_configuration.py index 843c7da089..51b1da0c6e 100644 --- a/tests/load/mssql/test_mssql_configuration.py +++ b/tests/load/mssql/test_mssql_configuration.py @@ -1,12 +1,10 @@ import os import struct -import pyodbc import pytest from dlt.common.configuration import ConfigFieldMissingException, resolve_configuration from dlt.common.configuration.exceptions import ConfigurationException -from dlt.common.exceptions import SystemConfigurationException from dlt.common.schema import Schema from dlt.common.utils import digest128 from dlt.destinations import mssql @@ -90,24 +88,14 @@ def test_mssql_fingerprint(connection_string: str, expected_fingerprint: str) -> def test_parse_native_representation() -> None: - # Case: unsupported driver specified. - with pytest.raises(SystemConfigurationException): - resolve_configuration( - MsSqlCredentials( - "mssql://test_user:test_pwd@sql.example.com/test_db?DRIVER=ODBC+Driver+13+for+SQL+Server" - ) - ) # Case: password not specified. with pytest.raises(ConfigFieldMissingException): - resolve_configuration( - MsSqlCredentials( - "mssql://test_user@sql.example.com/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server" - ) - ) + resolve_configuration(MsSqlCredentials("mssql://test_user@sql.example.com/test_db")) -def test_to_odbc_dsn_supported_driver_specified() -> None: - # Case: supported driver specified — ODBC Driver 18 for SQL Server. +def test_to_odbc_dsn() -> None: + # mssql-python bundles its own driver, so the DSN carries no DRIVER and any `driver` + # query parameter (legacy pyodbc config) is ignored. creds = resolve_configuration( MsSqlCredentials( "mssql://test_user:test_pwd@sql.example.com/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server" @@ -116,39 +104,19 @@ def test_to_odbc_dsn_supported_driver_specified() -> None: dsn = creds.to_odbc_dsn() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} assert result == { - "DRIVER": "ODBC Driver 18 for SQL Server", - "SERVER": "sql.example.com,1433", - "DATABASE": "test_db", - "UID": "test_user", - "PWD": "test_pwd", - } - - # Case: supported driver specified — ODBC Driver 17 for SQL Server. - creds = resolve_configuration( - MsSqlCredentials( - "mssql://test_user:test_pwd@sql.example.com/test_db?DRIVER=ODBC+Driver+17+for+SQL+Server" - ) - ) - dsn = creds.to_odbc_dsn() - result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} - assert result == { - "DRIVER": "ODBC Driver 17 for SQL Server", "SERVER": "sql.example.com,1433", "DATABASE": "test_db", "UID": "test_user", "PWD": "test_pwd", } - # Case: port and supported driver specified. + # Case: custom port. creds = resolve_configuration( - MsSqlCredentials( - "mssql://test_user:test_pwd@sql.example.com:12345/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server" - ) + MsSqlCredentials("mssql://test_user:test_pwd@sql.example.com:12345/test_db") ) dsn = creds.to_odbc_dsn() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} assert result == { - "DRIVER": "ODBC Driver 18 for SQL Server", "SERVER": "sql.example.com,12345", "DATABASE": "test_db", "UID": "test_user", @@ -157,16 +125,15 @@ def test_to_odbc_dsn_supported_driver_specified() -> None: def test_to_odbc_dsn_arbitrary_keys_specified() -> None: - # Case: arbitrary query keys (and supported driver) specified. + # Arbitrary query keys are passed through (the `driver` key is dropped). creds = resolve_configuration( MsSqlCredentials( - "mssql://test_user:test_pwd@sql.example.com:12345/test_db?FOO=a&BAR=b&DRIVER=ODBC+Driver+18+for+SQL+Server" + "mssql://test_user:test_pwd@sql.example.com:12345/test_db?FOO=a&BAR=b&Driver=ODBC+Driver+18+for+SQL+Server" ) ) dsn = creds.to_odbc_dsn() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} assert result == { - "DRIVER": "ODBC Driver 18 for SQL Server", "SERVER": "sql.example.com,12345", "DATABASE": "test_db", "UID": "test_user", @@ -175,48 +142,29 @@ def test_to_odbc_dsn_arbitrary_keys_specified() -> None: "BAR": "b", } - # Case: arbitrary capitalization. + +def test_to_odbc_dsn_connect_timeout_and_longasmax_dropped() -> None: + # mssql-python's connection-string parser rejects unknown keywords, so `connect_timeout` + # (passed via the connect() `timeout=` parameter instead) and `LongAsMax` (the driver + # handles long/max types natively) must never end up in the DSN. creds = resolve_configuration( MsSqlCredentials( - "mssql://test_user:test_pwd@sql.example.com:12345/test_db?FOO=a&bar=b&Driver=ODBC+Driver+18+for+SQL+Server" + "mssql://test_user:test_pwd@sql.example.com/test_db?connect_timeout=15&LongAsMax=yes&Encrypt=yes" ) ) dsn = creds.to_odbc_dsn() + assert "connect_timeout" not in dsn.lower() + assert "longasmax" not in dsn.lower() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} assert result == { - "DRIVER": "ODBC Driver 18 for SQL Server", - "SERVER": "sql.example.com,12345", + "SERVER": "sql.example.com,1433", "DATABASE": "test_db", "UID": "test_user", "PWD": "test_pwd", - "FOO": "a", - "BAR": "b", + "ENCRYPT": "yes", } -available_drivers = [d for d in pyodbc.drivers() if d in MsSqlCredentials.SUPPORTED_DRIVERS] - - -@pytest.mark.skipif(not available_drivers, reason="no supported driver available") -def test_to_odbc_dsn_driver_not_specified() -> None: - # Case: driver not specified, but supported driver is available. - creds = resolve_configuration( - MsSqlCredentials("mssql://test_user:test_pwd@sql.example.com/test_db") - ) - dsn = creds.to_odbc_dsn() - result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} - assert result in [ - { - "DRIVER": d, - "SERVER": "sql.example.com,1433", - "DATABASE": "test_db", - "UID": "test_user", - "PWD": "test_pwd", - } - for d in MsSqlCredentials.SUPPORTED_DRIVERS - ] - - # --------------------------------------------------------------------------- # Authentication methods # --------------------------------------------------------------------------- @@ -237,7 +185,6 @@ def _mssql_credentials(authentication: object = None, **kwargs: object) -> MsSql creds = MsSqlCredentials() creds.host = "sql.example.com" creds.database = "test_db" - creds.driver = "ODBC Driver 18 for SQL Server" # avoid probing for an installed driver if authentication is not None: creds.authentication = authentication # type: ignore[assignment] for key, value in kwargs.items(): @@ -372,7 +319,6 @@ def test_mssql_resolve_configuration_token_authentication() -> None: creds = MsSqlCredentials() creds.host = "sql.example.com" creds.database = "test_db" - creds.driver = "ODBC Driver 18 for SQL Server" creds.authentication = "ActiveDirectoryDeviceCode" resolved = resolve_configuration(creds) diff --git a/tests/load/mssql/test_mssql_docker_auth.py b/tests/load/mssql/test_mssql_docker_auth.py index 5a3f196566..2158ad8cd7 100644 --- a/tests/load/mssql/test_mssql_docker_auth.py +++ b/tests/load/mssql/test_mssql_docker_auth.py @@ -3,30 +3,34 @@ Start the server with `docker compose up` in this directory (see docker-compose.yml). Tests skip automatically when the server is unreachable, so they are safe to run anywhere. -These exercise the real `PyOdbcMsSqlClient.open_connection` code path. SQL Server in Docker -only supports SQL login (no Entra ID), so token/Azure AD authentication is validated against a -real Azure SQL / Fabric instance instead, via the parametrized `tests/load` suite. +These exercise the real mssql-python connection and load path. SQL Server in Docker only +supports SQL login (no Entra ID), so token/Azure AD authentication is validated against a real +Azure SQL / Fabric instance instead, via the parametrized `tests/load` suite. """ +from typing import Any + import pytest +import dlt from dlt.common.configuration import resolve_configuration from dlt.common.schema import Schema +from dlt.destinations import mssql from dlt.destinations.impl.mssql.configuration import MsSqlClientConfiguration, MsSqlCredentials # mark all tests as essential, do not remove pytestmark = pytest.mark.essential DOCKER_HOST = "localhost" -DOCKER_DATABASE = "master" DOCKER_USERNAME = "sa" DOCKER_PASSWORD = "Strong!Passw0rd" +TEST_DATABASE = "dlt_ci_test" -def _docker_credentials() -> MsSqlCredentials: +def _docker_credentials(database: str) -> MsSqlCredentials: creds = MsSqlCredentials() creds.host = DOCKER_HOST - creds.database = DOCKER_DATABASE + creds.database = database creds.username = DOCKER_USERNAME creds.password = DOCKER_PASSWORD creds.connect_timeout = 5 @@ -35,17 +39,29 @@ def _docker_credentials() -> MsSqlCredentials: return resolve_configuration(creds) -def _sql_client(creds: MsSqlCredentials): - from dlt.destinations import mssql - +def _sql_client(creds: MsSqlCredentials) -> Any: config = MsSqlClientConfiguration(credentials=creds)._bind_dataset_name("dataset") - client = mssql().client(Schema("schema"), config) - return client.sql_client + return mssql().client(Schema("schema"), config).sql_client + + +def _ensure_server_and_database() -> None: + """Open a real connection (skips if unreachable) and create the test database.""" + sql_client = _sql_client(_docker_credentials("master")) + try: + sql_client.open_connection() + except Exception as ex: # noqa: BLE001 + pytest.skip(f"Local SQL Server not reachable (run `docker compose up`): {ex}") + try: + sql_client.execute_sql( + f"IF DB_ID('{TEST_DATABASE}') IS NULL CREATE DATABASE {TEST_DATABASE}" + ) + finally: + sql_client.close_connection() def test_mssql_docker_sql_login_opens_connection() -> None: """SQL login (no Azure AD) opens a real connection - attrs_before is skipped.""" - creds = _docker_credentials() + creds = _docker_credentials("master") assert creds.to_odbc_attrs_before() is None sql_client = _sql_client(creds) @@ -59,3 +75,24 @@ def test_mssql_docker_sql_login_opens_connection() -> None: assert rows[0][0] == 1 finally: sql_client.close_connection() + + +def test_mssql_docker_pipeline_load() -> None: + """A full pipeline load round-trips data through the mssql-python driver.""" + _ensure_server_and_database() + + pipeline = dlt.pipeline( + pipeline_name="mssql_ci", + destination=mssql(credentials=_docker_credentials(TEST_DATABASE)), + dataset_name="mssql_ci_dataset", + dev_mode=True, + ) + pipeline.run( + [{"id": 1, "name": "a"}, {"id": 2, "name": "b"}, {"id": 3, "name": "c"}], + table_name="items", + ) + + with pipeline.sql_client() as client: + qualified = client.make_qualified_table_name("items") + rows = client.execute_sql(f"SELECT COUNT(*) FROM {qualified}") + assert rows[0][0] == 3 diff --git a/tests/load/synapse/test_synapse_configuration.py b/tests/load/synapse/test_synapse_configuration.py index ac052ffd8c..0669c65e2f 100644 --- a/tests/load/synapse/test_synapse_configuration.py +++ b/tests/load/synapse/test_synapse_configuration.py @@ -3,7 +3,6 @@ import pytest from dlt.common.configuration import resolve_configuration -from dlt.common.exceptions import SystemConfigurationException from dlt.common.schema import Schema from dlt.common.utils import digest128 from dlt.destinations import synapse @@ -82,18 +81,20 @@ def test_synapse_factory() -> None: assert client.capabilities.casefold_identifier is str -def test_parse_native_representation() -> None: - # Case: unsupported driver specified. - with pytest.raises(SystemConfigurationException): - resolve_configuration( - SynapseCredentials( - "synapse://test_user:test_pwd@test.sql.azuresynapse.net/test_db?DRIVER=ODBC+Driver+17+for+SQL+Server" - ) +def test_driver_query_parameter_is_ignored() -> None: + # mssql-python bundles its own driver, so a legacy `driver` query parameter is ignored + # and the DSN carries no DRIVER key. + creds = resolve_configuration( + SynapseCredentials( + "synapse://test_user:test_pwd@test.sql.azuresynapse.net/test_db?DRIVER=ODBC+Driver+17+for+SQL+Server" ) + ) + assert "DRIVER=" not in creds.to_odbc_dsn() -def test_to_odbc_dsn_longasmax() -> None: - # Case: LONGASMAX not specified in query (this is the expected scenario). +def test_to_odbc_dsn_longasmax_absent() -> None: + # The mssql-python driver handles long/max types natively, so LONGASMAX must never + # appear in the DSN, regardless of what the user passes in the query. creds = resolve_configuration( SynapseCredentials( "synapse://test_user:test_pwd@test.sql.azuresynapse.net/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server" @@ -101,14 +102,14 @@ def test_to_odbc_dsn_longasmax() -> None: ) dsn = creds.to_odbc_dsn() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} - assert result["LONGASMAX"] == "yes" + assert "LONGASMAX" not in result - # Case: LONGASMAX specified in query; specified value should be overridden. + # Case: LongAsMax specified in query; it is still dropped from the DSN. creds = resolve_configuration( SynapseCredentials( - "synapse://test_user:test_pwd@test.sql.azuresynapse.net/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server&LONGASMAX=no" + "synapse://test_user:test_pwd@test.sql.azuresynapse.net/test_db?DRIVER=ODBC+Driver+18+for+SQL+Server&LongAsMax=yes" ) ) dsn = creds.to_odbc_dsn() result = {k: v for k, v in (param.split("=") for param in dsn.split(";"))} - assert result["LONGASMAX"] == "yes" + assert "LONGASMAX" not in result diff --git a/uv.lock b/uv.lock index 9d6dc991d8..820c528526 100644 --- a/uv.lock +++ b/uv.lock @@ -2229,7 +2229,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "azure-identity", marker = "python_full_version < '3.13'" }, { name = "dbt-core", version = "1.7.19", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "pyodbc", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dc/f7/b94cc6672d50d0cef9f3910c49fd4a1ceebe26874fbd2b0ec95f39cd17cc/dbt-fabric-1.7.4.tar.gz", hash = "sha256:6f17f0ba683c2944c8f846589dca4b54106579af32ac5acafe701d1becc2496f", size = 25820, upload-time = "2024-02-02T21:00:42.189Z" } wheels = [ @@ -2384,7 +2384,7 @@ dependencies = [ { name = "azure-identity", marker = "python_full_version < '3.13'" }, { name = "dbt-core", version = "1.7.19", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "dbt-fabric", marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "pyodbc", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/97/d7244c6ab1ce380ec8ce453ecebe4692325028c7c2fe9863c5fbb163ca04/dbt-sqlserver-1.7.4.tar.gz", hash = "sha256:b9e85771a1c00e8f4aadefb37b00d02b3d49bc93ad7c52782fd9cae9db31dd98", size = 13386, upload-time = "2024-03-04T14:46:41.247Z" } wheels = [ @@ -2587,9 +2587,8 @@ ducklake = [ ] fabric = [ { name = "adlfs" }, + { name = "mssql-python" }, { name = "pyarrow" }, - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] filesystem = [ { name = "botocore" }, @@ -2633,8 +2632,7 @@ motherduck = [ { name = "pyarrow" }, ] mssql = [ - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "mssql-python" }, ] oracle = [ { name = "oracledb" }, @@ -2684,9 +2682,8 @@ sqlalchemy = [ ] synapse = [ { name = "adlfs" }, + { name = "mssql-python" }, { name = "pyarrow" }, - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] weaviate = [ { name = "weaviate-client" }, @@ -2867,6 +2864,9 @@ requires-dist = [ { name = "jsonpath-ng", specifier = ">=1.5.3,<1.8" }, { name = "lancedb", marker = "extra == 'lance'", specifier = ">=0.33.0" }, { name = "lancedb", marker = "extra == 'lancedb'", specifier = ">=0.22.0" }, + { name = "mssql-python", marker = "extra == 'fabric'", specifier = ">=1.4.0,!=1.7.0,!=1.7.1" }, + { name = "mssql-python", marker = "extra == 'mssql'", specifier = ">=1.4.0,!=1.7.0,!=1.7.1" }, + { name = "mssql-python", marker = "extra == 'synapse'", specifier = ">=1.4.0,!=1.7.0,!=1.7.1" }, { name = "oracledb", marker = "extra == 'oracle'", specifier = ">=3.4.1" }, { name = "orjson", marker = "python_full_version >= '3.14'", specifier = ">=3.11.0" }, { name = "orjson", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", specifier = ">=3.6.7,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,!=3.10.1,<4" }, @@ -2902,9 +2902,6 @@ requires-dist = [ { name = "pyiceberg", marker = "extra == 'pyiceberg'", specifier = ">=0.9.1" }, { name = "pyiceberg-core", marker = "extra == 'pyiceberg'", specifier = ">=0.6.0" }, { name = "pylance", marker = "extra == 'lance'", specifier = ">=6.0.1,<8" }, - { name = "pyodbc", marker = "extra == 'fabric'", specifier = ">=4.0.39" }, - { name = "pyodbc", marker = "extra == 'mssql'", specifier = ">=4.0.39" }, - { name = "pyodbc", marker = "extra == 'synapse'", specifier = ">=4.0.39" }, { name = "pytz", specifier = ">=2022.6" }, { name = "pywin32", marker = "sys_platform == 'win32'", specifier = ">=306" }, { name = "pyyaml", specifier = ">=5.4.1" }, @@ -4618,7 +4615,7 @@ mssql = [ { name = "pandas", marker = "python_full_version < '3.13'" }, { name = "pyarrow", marker = "python_full_version < '3.13'" }, { name = "pyarrow-hotfix", marker = "python_full_version < '3.13'" }, - { name = "pyodbc", version = "5.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "pyodbc", marker = "python_full_version < '3.13'" }, { name = "rich", marker = "python_full_version < '3.13'" }, ] postgres = [ @@ -6188,6 +6185,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/3e/c5187de84bb2c2ca334ab163fcacf19a23ebb1d876c837f81a1b324a15bf/msgspec-0.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:93f23528edc51d9f686808a361728e903d6f2be55c901d6f5c92e44c6d546bfc", size = 183011, upload-time = "2025-11-24T03:56:16.442Z" }, ] +[[package]] +name = "mssql-python" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-identity" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/59/0f54ac796a5dbe530602d3cdb2c747c371729f9872c4646588e79740207c/mssql_python-1.10.0-cp310-cp310-macosx_15_0_universal2.whl", hash = "sha256:65957b7f97974f0874cf8c5c3123e87f58d5a47a4d93931c9758a92c23f9f3b6", size = 28321559, upload-time = "2026-06-26T08:21:20.835Z" }, + { url = "https://files.pythonhosted.org/packages/84/04/aa0e47c7c8db56e4399c00ab47ba7c1d2afb02c59ffdc0f9f4996761b790/mssql_python-1.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:18c9b53c19737fa220b8fe95ad41078164be78eee676707e9262489e570504ec", size = 25305621, upload-time = "2026-06-26T08:21:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/45/0f/50816fde85d88488a9f44e87506ad174a3f410655f642564f51a1920aba8/mssql_python-1.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3cf7e2f11d92dad1caa07dfd28339a8ccfbe701b4de4fa77eb6bda0a80109b5", size = 25538078, upload-time = "2026-06-26T08:21:27.272Z" }, + { url = "https://files.pythonhosted.org/packages/f8/14/9e4f9a8003cf960ed5e91f046d3eb37b4ae8fbc8169a3fd218c102f5337f/mssql_python-1.10.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b245ad4b1008298e8b32218d869b2551d522604de72d5814b8112e0a14896a91", size = 25240036, upload-time = "2026-06-26T08:21:30.305Z" }, + { url = "https://files.pythonhosted.org/packages/25/bf/f7efba722b958185e2006ec79a39968a855d43458a5013e563abfde585f5/mssql_python-1.10.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fa168bdefaf07b3427b3f14c4c06c0cf5a196519c4bdff9cebec9705cf96154c", size = 25461927, upload-time = "2026-06-26T08:21:33.816Z" }, + { url = "https://files.pythonhosted.org/packages/f4/29/fcbd37364dfd572ff2e3ed3f4097177254e0c84ecee4f3998d8fe9abc841/mssql_python-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:0d418c3bfa9368ca24306abd7e79168acee9bdc0ecb2d8035698b9f3a9b3cb8e", size = 15522878, upload-time = "2026-06-26T08:21:36.935Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b6/3441bdb636a49d1c8b93d4087b7b99900728aa95ae17a7c59db323fbddce/mssql_python-1.10.0-cp311-cp311-macosx_15_0_universal2.whl", hash = "sha256:e2ecbde000f8b524413becc0ce5da511bbaf4b4b1e72d6d47076e82805f9f6b7", size = 28327662, upload-time = "2026-06-26T08:21:39.766Z" }, + { url = "https://files.pythonhosted.org/packages/48/5b/7d24f166667417d82fbad2d4baf753780598f639c858e7fc60cffb72d055/mssql_python-1.10.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e8a39d7806fcea8ecbaa5696cbe4238df09c3eec45a3816aaf0776987e9f14bf", size = 25805926, upload-time = "2026-06-26T08:21:42.678Z" }, + { url = "https://files.pythonhosted.org/packages/92/4b/76835ca0ef90dde26991eaf94e646cc958c0e2a828c9dbe962ef7709c2ab/mssql_python-1.10.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7c5fccd888274d6eaa0538c3d7f10892aa6f17c6581a04ca783fe44f720198ac", size = 26204276, upload-time = "2026-06-26T08:21:46.844Z" }, + { url = "https://files.pythonhosted.org/packages/9e/64/563c213aee5c235e455f5158e548f28c16549213582cfead78aa916b7b8e/mssql_python-1.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ab460f61baa24c6bff6a1a2de91dc75239b180672bdced1215ba5b37a7ba90a0", size = 25678503, upload-time = "2026-06-26T08:21:49.624Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c6/4cab32da5b7e6e8f390cc81fb5ba3935bcd93b79dda3d319151727d56f21/mssql_python-1.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3b875a1daa707c85076034508cec4f934a865cac9f4f98730b1a98716ec48cf9", size = 26055898, upload-time = "2026-06-26T08:21:52.472Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9b/ac067a24c484de34b525eafdf45814a970c7a3698790d8183a02c02cfde4/mssql_python-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:59f3a430d789c1b3b15b6e429cfa3412c78fb4ae756a11a0d871ab994bc96ef8", size = 15523008, upload-time = "2026-06-26T08:21:55.499Z" }, + { url = "https://files.pythonhosted.org/packages/5a/dc/a362f5321cfd75ea6f729f15c18f6e6c600a1f72026082c25df56ee49789/mssql_python-1.10.0-cp311-cp311-win_arm64.whl", hash = "sha256:ab4a9ac0b420f7b04cc4eedc418a403baf6f48f49f7752dc9aaa03fbb9be2d85", size = 18734312, upload-time = "2026-06-26T08:21:58.383Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/cc3e87cb4c6ea909f3e2d570396c88c216dd7360d84008d9d6e02e66d9b3/mssql_python-1.10.0-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:8949d73ccbecd5fad279a78b78ec522e474537e79ea03a5da40dd0fc28409546", size = 28326488, upload-time = "2026-06-26T08:22:00.886Z" }, + { url = "https://files.pythonhosted.org/packages/51/f7/90f9f7885648c5031c84aae7340554727087ae129c82ecda4f08a7530615/mssql_python-1.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa90c2b25f4b60cffcb07ab329b2e3cd01814c73c6890fb5a61ca2190fba2a85", size = 26299752, upload-time = "2026-06-26T08:22:03.98Z" }, + { url = "https://files.pythonhosted.org/packages/67/7d/83ac1f9c66d67ece0455144f38b1b53a24015af943679d2629382a26eecf/mssql_python-1.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c584689c720e92ce522f718c485cb18aa56f8cbb08da205d1d6b62176880f4fc", size = 26865344, upload-time = "2026-06-26T08:22:06.867Z" }, + { url = "https://files.pythonhosted.org/packages/6d/03/93074cba2046b7c65ed9161b8487bb3b591ee5caf7b519a803c0d7284d5d/mssql_python-1.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e594f74093cbd0ebf3a251cee1f04daac84655505a1ae512d98a54a4c60cb59e", size = 26107080, upload-time = "2026-06-26T08:22:10.023Z" }, + { url = "https://files.pythonhosted.org/packages/83/51/0f8474106671a70b084dbdb2993a4e88487b3547ebf0b66d63da8ae063da/mssql_python-1.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae16777095e35710c067147f2bcf41fcec9a1538a6df192731f15400fd3a4e4", size = 26645228, upload-time = "2026-06-26T08:22:12.712Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8e/0e0c75fd15fda8db965f517478f2152f56c3d3f0d830dd8358fc055e973e/mssql_python-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:74fef25e56d73d77965a3c727b51bb99c4fa25bde32e96e30bed3c6f77fb8117", size = 15519212, upload-time = "2026-06-26T08:22:15.279Z" }, + { url = "https://files.pythonhosted.org/packages/9a/eb/26c5043c5b7a1a0a292634ad65e33eeab151d9d069adc552bba63be80909/mssql_python-1.10.0-cp312-cp312-win_arm64.whl", hash = "sha256:38053e6d2dd8ae8ac096ee67d061dd6862ec164cec6bd67697e1f94d3c963966", size = 18729981, upload-time = "2026-06-26T08:22:18.426Z" }, + { url = "https://files.pythonhosted.org/packages/a7/29/dbdc757f4216bbb23f2768aec970c3c28ca7d61afde8e55fb6b25ffe0623/mssql_python-1.10.0-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:0e02f754d2c1641cf14e10068a833c0cc026ee7a7e839b92f20f32ee60726697", size = 28325494, upload-time = "2026-06-26T08:22:21.203Z" }, + { url = "https://files.pythonhosted.org/packages/57/59/32a16eb963899cf5cc39d04b27c17cc4785b136563d76167109c66137b48/mssql_python-1.10.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:00653ed1ea90d3fd32d3f41332823039eb2ba71b3bcffad6b1e6a1e9e89ddd92", size = 26803264, upload-time = "2026-06-26T08:22:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/2e/6a/c228df6993f3686e66d1cc8ad76db1a57241c2ba6ddd643a4bcce740c2eb/mssql_python-1.10.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:717bb19ebb67a520c49ed75af7d1748fdc5c468769f402155521d5a493ebd517", size = 27534316, upload-time = "2026-06-26T08:22:27.835Z" }, + { url = "https://files.pythonhosted.org/packages/86/e8/b65bce44a14584319ab167eb940bd3d292423225c9aef20b436e2959337f/mssql_python-1.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6955c20d3be5af9f3373c035cb141a52603a2c6406728521a28a10feee283bbf", size = 26546137, upload-time = "2026-06-26T08:22:31.196Z" }, + { url = "https://files.pythonhosted.org/packages/24/7c/5f23a21ad9e8e224575dee56f9cf661928cf1166bb1b7ab9ddc2a6d633ee/mssql_python-1.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cc1aca1243ed5ec994a5116d791d27bd45e8e0b1dfd8048b4eb9beb74ba203a", size = 27242014, upload-time = "2026-06-26T08:22:33.97Z" }, + { url = "https://files.pythonhosted.org/packages/df/89/11c1cb560112fda7c6cade09f2cd0fcf1fc91d94331e4458b7eb4e92ccd2/mssql_python-1.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:651d6873ace3782704b116912ec4e8e7d76fcdfec9941398c82c33dad2f17705", size = 15519505, upload-time = "2026-06-26T08:22:36.876Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2a/391a336b96d75b183462f06d59c490677a95f93183d1cea873b468af10de/mssql_python-1.10.0-cp313-cp313-win_arm64.whl", hash = "sha256:8bbcd14fa4f3ad9dfd3c26eee64825931322a6f8c74dd04fb70a0b23d7f4faa5", size = 18730229, upload-time = "2026-06-26T08:22:39.403Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/f8de0d278c6125d07f619d2b88a9e8742ea6f0037af9402116614d63f01d/mssql_python-1.10.0-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:31f30feda492de883e2440a1d2a6ec82dfc54d2f1ca64f90ed42a2918e30bca7", size = 28322330, upload-time = "2026-06-26T08:22:42.408Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/5fe873e0cb6572eb889c377d0597d070a41c3a8c78021e425c8dc12c4ef7/mssql_python-1.10.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:1de5847a975c92b6f2cd62bf1a4117f3068455679e5149ffb1b3a6c098bb3d10", size = 27310969, upload-time = "2026-06-26T08:22:45.678Z" }, + { url = "https://files.pythonhosted.org/packages/36/0f/8913684e667291dc5f861a5b979d5a5360c9d05b6215a20025702011f140/mssql_python-1.10.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:05a60e679b9e26cb53b99347f645489cd3ec6c7d4c1124fd08a740063267975f", size = 28205942, upload-time = "2026-06-26T08:22:48.712Z" }, + { url = "https://files.pythonhosted.org/packages/31/e6/db330ba8a7be7151246288bea68f71e86f1a708e534e70d8f24cfd3fcc45/mssql_python-1.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4b471ffee3149f2295104a98764b982145869d126aa59e084f8f35992decd125", size = 26989322, upload-time = "2026-06-26T08:22:51.994Z" }, + { url = "https://files.pythonhosted.org/packages/20/bc/876588646f20adcb6985b05f259a6f1bd1db17e19de9e547951f3bac19f8/mssql_python-1.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4aaa1fe032183a14bed0131120f3db5defe930563b97f9b0a08adc6d963b13e2", size = 27840470, upload-time = "2026-06-26T08:22:55.059Z" }, + { url = "https://files.pythonhosted.org/packages/1b/96/aca98e2ab26f67ad793c7a48e7424cce49833e6f220db1fbb2230b2b33c8/mssql_python-1.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:6d4ce7ce09c9f49918a39a4d57c262bb77f97da8b1f7efa489df837b070365c3", size = 16047610, upload-time = "2026-06-26T08:22:57.683Z" }, + { url = "https://files.pythonhosted.org/packages/4b/cf/fa65241815e4f748449a08400b3fa794f830e269888f7ceff4fdc3247c7e/mssql_python-1.10.0-cp314-cp314-win_arm64.whl", hash = "sha256:368cd690198ebb5fc26d2827d797e8be30cf729a0d2e18d255680aa6bc26417f", size = 19363204, upload-time = "2026-06-26T08:23:00.279Z" }, +] + [[package]] name = "multidict" version = "6.4.4" @@ -8463,14 +8504,6 @@ wheels = [ name = "pyodbc" version = "5.0.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version == '3.11.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "python_full_version < '3.11' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "(python_full_version == '3.12.*' and platform_python_implementation == 'PyPy') or (python_full_version == '3.12.*' and sys_platform == 'emscripten')", - "(python_full_version == '3.11.*' and platform_python_implementation == 'PyPy') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", - "(python_full_version < '3.11' and platform_python_implementation == 'PyPy') or (python_full_version < '3.11' and sys_platform == 'emscripten')", -] sdist = { url = "https://files.pythonhosted.org/packages/22/6f/012f32aecf744e439980257be0ba4dd8c70a4e03c9f86f5fcd986fbfb012/pyodbc-5.0.1.tar.gz", hash = "sha256:03d7d0b04d5a9156099ce8d03e92f3956783746fa9234eb6f5b5cfc12b645011", size = 115228, upload-time = "2023-10-13T17:08:30.737Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ce/97/337e82c5325f6cc039c5ed65d8b687c942b17763b88c4fbe21420b84ae4d/pyodbc-5.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9824b175db875a2dd116c7cf16dc3bdf14855404417afd145c5b839da222cb46", size = 72207, upload-time = "2023-10-13T17:07:44Z" }, @@ -8493,74 +8526,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/ef/2d24457682b4b239748f1d25c8a88b88ba99214fd78bedde54ff6186422c/pyodbc-5.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:3a3204ba4d1374fe8c301a76e04c9e9207d71cc346ca8282a85c20260d135b5d", size = 69250, upload-time = "2023-10-13T17:08:11.428Z" }, ] -[[package]] -name = "pyodbc" -version = "5.3.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten')", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", - "(python_full_version == '3.13.*' and platform_python_implementation == 'PyPy') or (python_full_version == '3.13.*' and sys_platform == 'emscripten')", -] -sdist = { url = "https://files.pythonhosted.org/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05", size = 121770, upload-time = "2025-10-17T18:04:09.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/cd/d0ac9e8963cf43f3c0e8ebd284cd9c5d0e17457be76c35abe4998b7b6df2/pyodbc-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6682cdec78f1302d0c559422c8e00991668e039ed63dece8bf99ef62173376a5", size = 71888, upload-time = "2025-10-17T18:02:58.285Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7b/95ea2795ea8a0db60414e14f117869a5ba44bd52387886c1a210da637315/pyodbc-5.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9cd3f0a9796b3e1170a9fa168c7e7ca81879142f30e20f46663b882db139b7d2", size = 71813, upload-time = "2025-10-17T18:02:59.722Z" }, - { url = "https://files.pythonhosted.org/packages/95/c9/6f4644b60af513ea1c9cab1ff4af633e8f300e8468f4ae3507f04524e641/pyodbc-5.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46185a1a7f409761716c71de7b95e7bbb004390c650d00b0b170193e3d6224bb", size = 318556, upload-time = "2025-10-17T18:03:01.129Z" }, - { url = "https://files.pythonhosted.org/packages/19/3f/24876d9cb9c6ce1bd2b6f43f69ebc00b8eb47bf1ed99ee95e340bf90ed79/pyodbc-5.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:349a9abae62a968b98f6bbd23d2825151f8d9de50b3a8f5f3271b48958fdb672", size = 322048, upload-time = "2025-10-17T18:03:02.522Z" }, - { url = "https://files.pythonhosted.org/packages/1f/27/faf17353605ac60f80136bc3172ed2d69d7defcb9733166293fc14ac2c52/pyodbc-5.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ac23feb7ddaa729f6b840639e92f83ff0ccaa7072801d944f1332cd5f5b05f47", size = 1286123, upload-time = "2025-10-17T18:03:04.157Z" }, - { url = "https://files.pythonhosted.org/packages/d4/61/c9d407d2aa3e89f9bb68acf6917b0045a788ae8c3f4045c34759cb77af63/pyodbc-5.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8aa396c6d6af52ccd51b8c8a5bffbb46fd44e52ce07ea4272c1d28e5e5b12722", size = 1343502, upload-time = "2025-10-17T18:03:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9f/f1b0f3238d873d4930aa2a2b8d5ba97132f6416764bf0c87368f8d6f2139/pyodbc-5.3.0-cp310-cp310-win32.whl", hash = "sha256:46869b9a6555ff003ed1d8ebad6708423adf2a5c88e1a578b9f029fb1435186e", size = 62968, upload-time = "2025-10-17T18:03:06.933Z" }, - { url = "https://files.pythonhosted.org/packages/d8/26/5f8ebdca4735aad0119aaaa6d5d73b379901b7a1dbb643aaa636040b27cf/pyodbc-5.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:705903acf6f43c44fc64e764578d9a88649eb21bf7418d78677a9d2e337f56f2", size = 69397, upload-time = "2025-10-17T18:03:08.49Z" }, - { url = "https://files.pythonhosted.org/packages/d1/c8/480a942fd2e87dd7df6d3c1f429df075695ed8ae34d187fe95c64219fd49/pyodbc-5.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:c68d9c225a97aedafb7fff1c0e1bfe293093f77da19eaf200d0e988fa2718d16", size = 64446, upload-time = "2025-10-17T18:03:09.333Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c7/534986d97a26cb8f40ef456dfcf00d8483161eade6d53fa45fcf2d5c2b87/pyodbc-5.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebc3be93f61ea0553db88589e683ace12bf975baa954af4834ab89f5ee7bf8ae", size = 71958, upload-time = "2025-10-17T18:03:10.163Z" }, - { url = "https://files.pythonhosted.org/packages/69/3c/6fe3e9eae6db1c34d6616a452f9b954b0d5516c430f3dd959c9d8d725f2a/pyodbc-5.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9b987a25a384f31e373903005554230f5a6d59af78bce62954386736a902a4b3", size = 71843, upload-time = "2025-10-17T18:03:11.058Z" }, - { url = "https://files.pythonhosted.org/packages/44/0e/81a0315d0bf7e57be24338dbed616f806131ab706d87c70f363506dc13d5/pyodbc-5.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:676031723aac7dcbbd2813bddda0e8abf171b20ec218ab8dfb21d64a193430ea", size = 327191, upload-time = "2025-10-17T18:03:11.93Z" }, - { url = "https://files.pythonhosted.org/packages/43/ae/b95bb2068f911950322a97172c68675c85a3e87dc04a98448c339fcbef21/pyodbc-5.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5c30c5cd40b751f77bbc73edd32c4498630939bcd4e72ee7e6c9a4b982cc5ca", size = 332228, upload-time = "2025-10-17T18:03:13.096Z" }, - { url = "https://files.pythonhosted.org/packages/dc/21/2433625f7d5922ee9a34e3805805fa0f1355d01d55206c337bb23ec869bf/pyodbc-5.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2035c7dfb71677cd5be64d3a3eb0779560279f0a8dc6e33673499498caa88937", size = 1296469, upload-time = "2025-10-17T18:03:14.61Z" }, - { url = "https://files.pythonhosted.org/packages/3a/f4/c760caf7bb9b3ab988975d84bd3e7ebda739fe0075c82f476d04ee97324c/pyodbc-5.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5cbe4d753723c8a8f65020b7a259183ef5f14307587165ce37e8c7e251951852", size = 1353163, upload-time = "2025-10-17T18:03:16.272Z" }, - { url = "https://files.pythonhosted.org/packages/14/ad/f9ca1e9e44fd91058f6e35b233b1bb6213d590185bfcc2a2c4f1033266e7/pyodbc-5.3.0-cp311-cp311-win32.whl", hash = "sha256:d255f6b117d05cfc046a5201fdf39535264045352ea536c35777cf66d321fbb8", size = 62925, upload-time = "2025-10-17T18:03:17.649Z" }, - { url = "https://files.pythonhosted.org/packages/e6/cf/52b9b94efd8cfd11890ae04f31f50561710128d735e4e38a8fbb964cd2c2/pyodbc-5.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:f1ad0e93612a6201621853fc661209d82ff2a35892b7d590106fe8f97d9f1f2a", size = 69329, upload-time = "2025-10-17T18:03:18.474Z" }, - { url = "https://files.pythonhosted.org/packages/8b/6f/bf5433bb345007f93003fa062e045890afb42e4e9fc6bd66acc2c3bd12ca/pyodbc-5.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:0df7ff47fab91ea05548095b00e5eb87ed88ddf4648c58c67b4db95ea4913e23", size = 64447, upload-time = "2025-10-17T18:03:19.691Z" }, - { url = "https://files.pythonhosted.org/packages/f5/0c/7ecf8077f4b932a5d25896699ff5c394ffc2a880a9c2c284d6a3e6ea5949/pyodbc-5.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ebf6b5d989395efe722b02b010cb9815698a4d681921bf5db1c0e1195ac1bde", size = 72994, upload-time = "2025-10-17T18:03:20.551Z" }, - { url = "https://files.pythonhosted.org/packages/03/78/9fbde156055d88c1ef3487534281a5b1479ee7a2f958a7e90714968749ac/pyodbc-5.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:197bb6ddafe356a916b8ee1b8752009057fce58e216e887e2174b24c7ab99269", size = 72535, upload-time = "2025-10-17T18:03:21.423Z" }, - { url = "https://files.pythonhosted.org/packages/9f/f9/8c106dcd6946e95fee0da0f1ba58cd90eb872eebe8968996a2ea1f7ac3c1/pyodbc-5.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6ccb5315ec9e081f5cbd66f36acbc820ad172b8fa3736cf7f993cdf69bd8a96", size = 333565, upload-time = "2025-10-17T18:03:22.695Z" }, - { url = "https://files.pythonhosted.org/packages/4b/30/2c70f47a76a4fafa308d148f786aeb35a4d67a01d41002f1065b465d9994/pyodbc-5.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5dd3d5e469f89a3112cf8b0658c43108a4712fad65e576071e4dd44d2bd763c7", size = 340283, upload-time = "2025-10-17T18:03:23.691Z" }, - { url = "https://files.pythonhosted.org/packages/7d/b2/0631d84731606bfe40d3b03a436b80cbd16b63b022c7b13444fb30761ca8/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b180bc5e49b74fd40a24ef5b0fe143d0c234ac1506febe810d7434bf47cb925b", size = 1302767, upload-time = "2025-10-17T18:03:25.311Z" }, - { url = "https://files.pythonhosted.org/packages/74/b9/707c5314cca9401081b3757301241c167a94ba91b4bd55c8fa591bf35a4a/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e3c39de3005fff3ae79246f952720d44affc6756b4b85398da4c5ea76bf8f506", size = 1361251, upload-time = "2025-10-17T18:03:26.538Z" }, - { url = "https://files.pythonhosted.org/packages/97/7c/893036c8b0c8d359082a56efdaa64358a38dda993124162c3faa35d1924d/pyodbc-5.3.0-cp312-cp312-win32.whl", hash = "sha256:d32c3259762bef440707098010035bbc83d1c73d81a434018ab8c688158bd3bb", size = 63413, upload-time = "2025-10-17T18:03:27.903Z" }, - { url = "https://files.pythonhosted.org/packages/c0/70/5e61b216cc13c7f833ef87f4cdeab253a7873f8709253f5076e9bb16c1b3/pyodbc-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe77eb9dcca5fc1300c9121f81040cc9011d28cff383e2c35416e9ec06d4bc95", size = 70133, upload-time = "2025-10-17T18:03:28.746Z" }, - { url = "https://files.pythonhosted.org/packages/aa/85/e7d0629c9714a85eb4f85d21602ce6d8a1ec0f313fde8017990cf913e3b4/pyodbc-5.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:afe7c4ac555a8d10a36234788fc6cfc22a86ce37fc5ba88a1f75b3e6696665dc", size = 64700, upload-time = "2025-10-17T18:03:29.638Z" }, - { url = "https://files.pythonhosted.org/packages/0c/1d/9e74cbcc1d4878553eadfd59138364b38656369eb58f7e5b42fb344c0ce7/pyodbc-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e9ab0b91de28a5ab838ac4db0253d7cc8ce2452efe4ad92ee6a57b922bf0c24", size = 72975, upload-time = "2025-10-17T18:03:30.466Z" }, - { url = "https://files.pythonhosted.org/packages/37/c7/27d83f91b3144d3e275b5b387f0564b161ddbc4ce1b72bb3b3653e7f4f7a/pyodbc-5.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6132554ffbd7910524d643f13ce17f4a72f3a6824b0adef4e9a7f66efac96350", size = 72541, upload-time = "2025-10-17T18:03:31.348Z" }, - { url = "https://files.pythonhosted.org/packages/1b/33/2bb24e7fc95e98a7b11ea5ad1f256412de35d2e9cc339be198258c1d9a76/pyodbc-5.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1629af4706e9228d79dabb4863c11cceb22a6dab90700db0ef449074f0150c0d", size = 343287, upload-time = "2025-10-17T18:03:32.287Z" }, - { url = "https://files.pythonhosted.org/packages/fa/24/88cde8b6dc07a93a92b6c15520a947db24f55db7bd8b09e85956642b7cf3/pyodbc-5.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ceaed87ba2ea848c11223f66f629ef121f6ebe621f605cde9cfdee4fd9f4b68", size = 350094, upload-time = "2025-10-17T18:03:33.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/99/53c08562bc171a618fa1699297164f8885e66cde38c3b30f454730d0c488/pyodbc-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3cc472c8ae2feea5b4512e23b56e2b093d64f7cbc4b970af51da488429ff7818", size = 1301029, upload-time = "2025-10-17T18:03:34.561Z" }, - { url = "https://files.pythonhosted.org/packages/d8/10/68a0b5549876d4b53ba4c46eed2a7aca32d589624ed60beef5bd7382619e/pyodbc-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c79df54bbc25bce9f2d87094e7b39089c28428df5443d1902b0cc5f43fd2da6f", size = 1361420, upload-time = "2025-10-17T18:03:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/41/0f/9dfe4987283ffcb981c49a002f0339d669215eb4a3fe4ee4e14537c52852/pyodbc-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c2eb0b08e24fe5c40c7ebe9240c5d3bd2f18cd5617229acee4b0a0484dc226f2", size = 63399, upload-time = "2025-10-17T18:03:36.931Z" }, - { url = "https://files.pythonhosted.org/packages/56/03/15dcefe549d3888b649652af7cca36eda97c12b6196d92937ca6d11306e9/pyodbc-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:01166162149adf2b8a6dc21a212718f205cabbbdff4047dc0c415af3fd85867e", size = 70133, upload-time = "2025-10-17T18:03:38.47Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c1/c8b128ae59a14ecc8510e9b499208e342795aecc3af4c3874805c720b8db/pyodbc-5.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:363311bd40320b4a61454bebf7c38b243cd67c762ed0f8a5219de3ec90c96353", size = 64683, upload-time = "2025-10-17T18:03:39.68Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f2/c26d82a7ce1e90b8bbb8731d3d53de73814e2f6606b9db9d978303aa8d5f/pyodbc-5.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3f1bdb3ce6480a17afaaef4b5242b356d4997a872f39e96f015cabef00613797", size = 73513, upload-time = "2025-10-17T18:03:40.536Z" }, - { url = "https://files.pythonhosted.org/packages/82/d5/1ab1b7c4708cbd701990a8f7183c5bb5e0712d5e8479b919934e46dadab4/pyodbc-5.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7713c740a10f33df3cb08f49a023b7e1e25de0c7c99650876bbe717bc95ee780", size = 72631, upload-time = "2025-10-17T18:03:41.713Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/7e3831eeac2b09b31a77e6b3495491ce162035ff2903d7261b49d35aa3c2/pyodbc-5.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf18797a12e70474e1b7f5027deeeccea816372497e3ff2d46b15bec2d18a0cc", size = 344580, upload-time = "2025-10-17T18:03:42.67Z" }, - { url = "https://files.pythonhosted.org/packages/a2/a6/71d26d626a3c45951620b7ff356ec920e420f0e09b0a924123682aa5e4ab/pyodbc-5.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:08b2439500e212625471d32f8fde418075a5ddec556e095e5a4ba56d61df2dc6", size = 350224, upload-time = "2025-10-17T18:03:43.731Z" }, - { url = "https://files.pythonhosted.org/packages/93/14/f702c5e8c2d595776266934498505f11b7f1545baf21ffec1d32c258e9d3/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:729c535341bb09c476f219d6f7ab194bcb683c4a0a368010f1cb821a35136f05", size = 1301503, upload-time = "2025-10-17T18:03:45.013Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b2/ad92ebdd1b5c7fec36b065e586d1d34b57881e17ba5beec5c705f1031058/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c67e7f2ce649155ea89beb54d3b42d83770488f025cf3b6f39ca82e9c598a02e", size = 1361050, upload-time = "2025-10-17T18:03:46.298Z" }, - { url = "https://files.pythonhosted.org/packages/19/40/dc84e232da07056cb5aaaf5f759ba4c874bc12f37569f7f1670fc71e7ae1/pyodbc-5.3.0-cp314-cp314-win32.whl", hash = "sha256:a48d731432abaee5256ed6a19a3e1528b8881f9cb25cb9cf72d8318146ea991b", size = 65670, upload-time = "2025-10-17T18:03:56.414Z" }, - { url = "https://files.pythonhosted.org/packages/b8/79/c48be07e8634f764662d7a279ac204f93d64172162dbf90f215e2398b0bd/pyodbc-5.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:58635a1cc859d5af3f878c85910e5d7228fe5c406d4571bffcdd281375a54b39", size = 72177, upload-time = "2025-10-17T18:03:57.296Z" }, - { url = "https://files.pythonhosted.org/packages/fc/79/e304574446b2263f428ce14df590ba52c2e0e0205e8d34b235b582b7d57e/pyodbc-5.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:754d052030d00c3ac38da09ceb9f3e240e8dd1c11da8906f482d5419c65b9ef5", size = 66668, upload-time = "2025-10-17T18:03:58.174Z" }, - { url = "https://files.pythonhosted.org/packages/43/17/f4eabf443b838a2728773554017d08eee3aca353102934a7e3ba96fb0e31/pyodbc-5.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f927b440c38ade1668f0da64047ffd20ec34e32d817f9a60d07553301324b364", size = 75780, upload-time = "2025-10-17T18:03:47.273Z" }, - { url = "https://files.pythonhosted.org/packages/59/ea/e79e168c3d38c27d59d5d96273fd9e3c3ba55937cc944c4e60618f51de90/pyodbc-5.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:25c4cfb2c08e77bc6e82f666d7acd52f0e52a0401b1876e60f03c73c3b8aedc0", size = 75503, upload-time = "2025-10-17T18:03:48.171Z" }, - { url = "https://files.pythonhosted.org/packages/90/81/d1d7c125ec4a20e83fdc28e119b8321192b2bd694f432cf63e1199b2b929/pyodbc-5.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc834567c2990584b9726cba365834d039380c9dbbcef3030ddeb00c6541b943", size = 398356, upload-time = "2025-10-17T18:03:49.131Z" }, - { url = "https://files.pythonhosted.org/packages/5e/fc/f6be4b3cc3910f8c2aba37aa41671121fd6f37b402ae0fefe53a70ac7cd5/pyodbc-5.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8339d3094858893c1a68ee1af93efc4dff18b8b65de54d99104b99af6306320d", size = 397291, upload-time = "2025-10-17T18:03:50.18Z" }, - { url = "https://files.pythonhosted.org/packages/03/2e/0610b1ed05a5625528d52f6cece9610e84617d35f475c89c2a52f66d13f7/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74528fe148980d0c735c0ebb4a4dc74643ac4574337c43c1006ac4d09593f92d", size = 1353900, upload-time = "2025-10-17T18:03:51.339Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f1/43497e1d37f9f71b43b2b3172e7b1bdf50851e278390c3fb6b46a3630c53/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d89a7f2e24227150c13be8164774b7e1f9678321a4248f1356a465b9cc17d31e", size = 1406062, upload-time = "2025-10-17T18:03:52.546Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8b/88a1277c2f7d9ab1cec0a71e074ba24fd4a1710a43974682546da90a1343/pyodbc-5.3.0-cp314-cp314t-win32.whl", hash = "sha256:af4d8c9842fc4a6360c31c35508d6594d5a3b39922f61b282c2b4c9d9da99514", size = 70132, upload-time = "2025-10-17T18:03:53.715Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c7/ee98c62050de4aa8bafb6eb1e11b95e0b0c898bd5930137c6dc776e06a9b/pyodbc-5.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bfeb3e34795d53b7d37e66dd54891d4f9c13a3889a8f5fe9640e56a82d770955", size = 79452, upload-time = "2025-10-17T18:03:54.664Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8f/d8889efd96bbe8e5d43ff9701f6b1565a8e09c3e1f58c388d550724f777b/pyodbc-5.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:13656184faa3f2d5c6f19b701b8f247342ed581484f58bf39af7315c054e69db", size = 70142, upload-time = "2025-10-17T18:03:55.551Z" }, -] - [[package]] name = "pyopenssl" version = "25.1.0" From 2d216f3006e2f33950a9f7619b73e470046e8049 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Wed, 1 Jul 2026 23:27:34 +0200 Subject: [PATCH 4/4] feat(mssql): pass all Entra ID authentication methods straight through to mssql-python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mssql-python signs in for every supported Entra ID authentication method itself, so dlt no longer needs to acquire or inject any token for named `authentication` methods: - Removed the azure-identity injection machinery for authentication methods entirely: `AZURE_IDENTITY_INJECTION_AUTHENTICATION`, `create_token_credential`, `get_token_credential`, `uses_token_authentication`, `setup_token_credential`, and the Service Principal-without-secret fallback to `DefaultAzureCredential`. `attrs_before` is never built for a named method anymore. - `DRIVER_NATIVE_AUTHENTICATION` becomes a single `SUPPORTED_AUTHENTICATION` set: every one of the seven `ActiveDirectory*` methods is validated the same way and written straight to `Authentication=` in the DSN. The `default` alias still normalizes to `ActiveDirectoryDefault` before being written, since mssql-python only recognizes the canonical name. - A Service Principal without a secret is no longer silently redirected to `DefaultAzureCredential` — it's passed through like any other method, and mssql-python is left to accept or reject it. - `build_token_attrs_before`/`to_odbc_attrs_before` are kept (always returning `None` today) rather than removed, as the injection point for a future explicit access-token feature. - Comments and docs no longer describe a native-vs-injected split or reference pyodbc; there is one supported-methods list, and mssql-python owns the sign-in for all of it. Updates the mssql/fabric/synapse destination docs and the offline mssql/fabric configuration unit tests to match. --- dlt/destinations/impl/fabric/configuration.py | 37 ++--- dlt/destinations/impl/mssql/configuration.py | 143 ++++-------------- dlt/destinations/impl/mssql/sql_client.py | 11 +- .../docs/dlt-ecosystem/destinations/fabric.md | 30 ++-- .../docs/dlt-ecosystem/destinations/mssql.md | 31 ++-- .../dlt-ecosystem/destinations/synapse.md | 3 +- .../load/fabric/test_fabric_configuration.py | 117 +++++++------- tests/load/mssql/test_mssql_configuration.py | 112 ++++++++------ 8 files changed, 194 insertions(+), 290 deletions(-) diff --git a/dlt/destinations/impl/fabric/configuration.py b/dlt/destinations/impl/fabric/configuration.py index 3377f1af5a..af78148317 100644 --- a/dlt/destinations/impl/fabric/configuration.py +++ b/dlt/destinations/impl/fabric/configuration.py @@ -9,12 +9,10 @@ from dlt.destinations.impl.mssql.configuration import ( apply_authentication_to_dsn, build_token_attrs_before, - setup_token_credential, validate_authentication, ) -# Fabric Warehouse only supports Entra ID authentication, so it defaults to Service Principal -# (the shared MsSql auth machinery additionally enables azure-identity token methods). +# Fabric Warehouse only supports Entra ID authentication, so it defaults to Service Principal. _DEFAULT_AUTHENTICATION = "ActiveDirectoryServicePrincipal" @@ -23,16 +21,10 @@ class FabricCredentials(AzureServicePrincipalCredentials): """Credentials for Microsoft Fabric Warehouse. Supports several Entra ID authentication methods, selected through `authentication`: - - * **Driver-native** (the ODBC driver authenticates): `ActiveDirectoryServicePrincipal` - (default), `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, - `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`. - * **azure-identity** (dlt acquires an access token and injects it, works cross-platform): - `ActiveDirectoryDefault` (alias `default`, uses `DefaultAzureCredential`), - `ActiveDirectoryDeviceCode` (uses `DeviceCodeCredential`). - - When `authentication` is left at its default but no Service Principal secret is configured, - dlt falls back to `ActiveDirectoryDefault` and injects its token. + `ActiveDirectoryServicePrincipal` (default), `ActiveDirectoryPassword`, + `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, + `ActiveDirectoryDefault` (alias `default`), `ActiveDirectoryDeviceCode`. All are passed + straight through as `Authentication=` in the DSN — mssql-python performs the sign-in. Inherits from AzureServicePrincipalCredentials for the Service Principal fields. """ @@ -50,10 +42,10 @@ class FabricCredentials(AzureServicePrincipalCredentials): """Connection timeout in seconds (default: 15)""" authentication: str = _DEFAULT_AUTHENTICATION - """Authentication method. Driver-native: `ActiveDirectoryServicePrincipal` (default), - `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, - `ActiveDirectoryMsi`. azure-identity (token injected by dlt): `ActiveDirectoryDefault` - (alias `default`), `ActiveDirectoryDeviceCode`.""" + """Authentication method, passed straight through as `Authentication=` in the DSN: + `ActiveDirectoryServicePrincipal` (default), `ActiveDirectoryPassword`, + `ActiveDirectoryIntegrated`, `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, + `ActiveDirectoryDefault` (alias `default`), `ActiveDirectoryDeviceCode`.""" username: str | None = None """User principal name, used with `ActiveDirectoryPassword` authentication.""" @@ -66,13 +58,12 @@ class FabricCredentials(AzureServicePrincipalCredentials): """Not used for Fabric Warehouse credentials (only staging credentials need this)""" def on_partial(self) -> None: - """Set up token-based credentials and resolve once host and database are known. + """Resolve once host and database are known. - Token-based methods (and the default Service Principal method without a secret) get an - azure-identity credential whose token is injected into the connection. Driver-native - methods need no token and resolve as-is. Auth logic is shared with `MsSqlCredentials`. + Fabric always has an `authentication` value set (default: Service Principal), so this + resolves as soon as the connection target is known. `on_resolved` validates. Auth logic + is shared with `MsSqlCredentials`. """ - setup_token_credential(self) # Resolve if we have the warehouse connection details (not the storage account name) if self.host and self.database: self.resolve() @@ -98,7 +89,7 @@ def get_odbc_dsn_dict(self) -> Dict[str, Any]: return params def to_odbc_attrs_before(self) -> dict[int, bytes] | None: - """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + """Return `attrs_before` with a directly injected Entra ID access token, or None.""" return build_token_attrs_before(self) def to_odbc_dsn(self) -> str: diff --git a/dlt/destinations/impl/mssql/configuration.py b/dlt/destinations/impl/mssql/configuration.py index a624dab588..c592f36950 100644 --- a/dlt/destinations/impl/mssql/configuration.py +++ b/dlt/destinations/impl/mssql/configuration.py @@ -1,45 +1,29 @@ import dataclasses -import struct -from typing import ClassVar, Any, Final, List, Dict, Optional, TYPE_CHECKING +from typing import ClassVar, Any, Final, List, Dict, Optional -from dlt import version from dlt.common.configuration import configspec from dlt.common.configuration.exceptions import ConfigurationException from dlt.common.configuration.specs import ConnectionStringCredentials, CredentialsWithDefault from dlt.common.typing import TSecretStrValue -from dlt.common.exceptions import MissingDependencyException from dlt.common.destination.client import DestinationClientDwhWithStagingConfiguration from dlt.common.utils import digest128 -if TYPE_CHECKING: - from azure.core.credentials import TokenCredential - -_AZURE_AUTH_EXTRA = f"{version.DLT_PKG_NAME}[az]" - -# ODBC connection attribute used to inject a pre-acquired Entra ID access token. +# ODBC attribute used to inject a pre-acquired Entra ID access token, kept for a future explicit +# access-token feature. Not used by any authentication method today. # https://learn.microsoft.com/sql/connect/odbc/using-azure-active-directory#authenticating-with-an-access-token SQL_COPT_SS_ACCESS_TOKEN = 1256 SQL_TOKEN_SCOPE = "https://database.windows.net/.default" -# Authentication methods the ODBC driver performs natively (passed as `Authentication=` in the -# DSN). Poolable: the driver signs in itself, so no token needs to be reacquired and injected -# per connection. -DRIVER_NATIVE_AUTHENTICATION = frozenset( +# Entra ID authentication methods supported by mssql-python's `Authentication=` connection +# option. mssql-python performs the sign-in for all of them; dlt only builds the DSN. +SUPPORTED_AUTHENTICATION = frozenset( { "ActiveDirectoryServicePrincipal", "ActiveDirectoryPassword", "ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", "ActiveDirectoryMsi", - } -) - -# Authentication methods pyodbc/msodbcsql does not support natively. dlt acquires the access -# token itself with azure-identity and injects it into the connection (`attrs_before`), so these -# work cross-platform but are not poolable. -AZURE_IDENTITY_INJECTION_AUTHENTICATION = frozenset( - { "ActiveDirectoryDefault", "ActiveDirectoryDeviceCode", } @@ -56,98 +40,16 @@ def _normalize_authentication(authentication: str) -> str: return _AUTHENTICATION_ALIASES.get(authentication.lower(), authentication) -def create_token_credential(authentication: str) -> "TokenCredential": - """Create an azure-identity credential for a token-injection authentication method. - - Expects the canonical name (`ActiveDirectoryDefault` or `ActiveDirectoryDeviceCode`), as - already resolved by `_normalize_authentication`. - """ - try: - from azure.identity import DefaultAzureCredential, DeviceCodeCredential - except ModuleNotFoundError: - raise MissingDependencyException("MsSqlCredentials", [_AZURE_AUTH_EXTRA]) - - credential_factories = { - "ActiveDirectoryDefault": DefaultAzureCredential, - "ActiveDirectoryDeviceCode": DeviceCodeCredential, - } - return credential_factories[authentication]() # type: ignore[no-any-return] - - -def uses_token_authentication(credentials: Any) -> bool: - """True when dlt must acquire an access token and inject it into the connection. - - Derived from the configured method, not from `has_default_credentials`: cooperative - `on_partial` calls up the MRO may set a default credential even for driver-native methods. - """ - authentication = credentials.authentication - if not authentication: - return False - if _normalize_authentication(authentication) in AZURE_IDENTITY_INJECTION_AUTHENTICATION: - return True - # ActiveDirectoryServicePrincipal without a secret cannot authenticate through the ODBC - # driver, so dlt falls back to a DefaultAzureCredential token. - return authentication == "ActiveDirectoryServicePrincipal" and not ( - credentials.azure_client_id - and credentials.azure_client_secret - and credentials.azure_tenant_id - ) - - -def setup_token_credential(credentials: Any) -> None: - """Create and store the azure-identity credential for token-injection authentication.""" - authentication = credentials.authentication or "" - normalized = _normalize_authentication(authentication) - if normalized in AZURE_IDENTITY_INJECTION_AUTHENTICATION: - credentials._set_default_credentials(create_token_credential(normalized)) - elif authentication == "ActiveDirectoryServicePrincipal" and not ( - credentials.azure_client_id - and credentials.azure_client_secret - and credentials.azure_tenant_id - ): - credentials._set_default_credentials(create_token_credential("ActiveDirectoryDefault")) - - -def get_token_credential(credentials: Any) -> "TokenCredential": - """Return the azure-identity credential used for token authentication. - - Reuses the credential created during resolution (so azure-identity can cache tokens - across connections) and creates one on demand otherwise. - """ - if credentials.has_default_credentials(): - return credentials.default_credentials() # type: ignore[no-any-return] - authentication = _normalize_authentication(credentials.authentication or "") - if authentication not in AZURE_IDENTITY_INJECTION_AUTHENTICATION: - authentication = "ActiveDirectoryDefault" - return create_token_credential(authentication) - - -def build_token_attrs_before(credentials: Any) -> dict[int, bytes] | None: - """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" - if not uses_token_authentication(credentials): - return None - token = get_token_credential(credentials).get_token(SQL_TOKEN_SCOPE).token - encoded_token = token.encode("utf-16-le") - token_struct = struct.pack(f" None: """Validate the configured authentication method.""" authentication = credentials.authentication if not authentication: return # plain SQL login (username/password) normalized = _normalize_authentication(authentication) - if ( - normalized not in DRIVER_NATIVE_AUTHENTICATION - and normalized not in AZURE_IDENTITY_INJECTION_AUTHENTICATION - ): - supported = sorted(DRIVER_NATIVE_AUTHENTICATION) + sorted( - AZURE_IDENTITY_INJECTION_AUTHENTICATION - ) + if normalized not in SUPPORTED_AUTHENTICATION: raise ConfigurationException( f"Unsupported `authentication` method `{authentication}`." - f" Supported methods: {', '.join(supported)}." + f" Supported methods: {', '.join(sorted(SUPPORTED_AUTHENTICATION))}." ) if authentication == "ActiveDirectoryPassword" and not ( credentials.username and credentials.password @@ -159,15 +61,15 @@ def validate_authentication(credentials: Any) -> None: def apply_authentication_to_dsn(credentials: Any, params: dict[str, Any]) -> None: """Add UID/PWD/Authentication keys to an ODBC DSN dict based on the authentication method.""" - if uses_token_authentication(credentials): - # The access token is injected via attrs_before, so the DSN carries no credentials. - return authentication = credentials.authentication if not authentication: # Plain SQL login. params["UID"] = credentials.username params["PWD"] = credentials.password return + # Write the canonical name, not the thin `default` alias — mssql-python only recognizes the + # canonical `ActiveDirectory*` values in the `Authentication=` DSN keyword. + authentication = _normalize_authentication(authentication) params["AUTHENTICATION"] = authentication if ( authentication == "ActiveDirectoryServicePrincipal" @@ -186,6 +88,15 @@ def apply_authentication_to_dsn(credentials: Any, params: dict[str, Any]) -> Non params["PWD"] = credentials.password +def build_token_attrs_before(credentials: Any) -> dict[int, bytes] | None: + """Return `attrs_before` with a directly injected Entra ID access token, or None. + + mssql-python performs the sign-in for every supported authentication method itself, so there + is nothing to inject here today. Kept as the hook for a future explicit access-token feature. + """ + return None + + def escape_mssql_odbc_value(value: Optional[str]) -> str: """Escape a value for MSSQL ADO/ODBC connection string format. @@ -239,11 +150,10 @@ class MsSqlCredentials(ConnectionStringCredentials, CredentialsWithDefault): authentication: str | None = None """Authentication method. Empty (default) uses plain SQL login (`username`/`password`). - Driver-native (passed as `Authentication=` in the DSN, poolable): + Supported Entra ID methods, passed straight through as `Authentication=` in the DSN: `ActiveDirectoryServicePrincipal`, `ActiveDirectoryPassword`, `ActiveDirectoryIntegrated`, - `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`. azure-identity (token acquired and - injected by dlt, not poolable): `ActiveDirectoryDefault` (alias `default`), - `ActiveDirectoryDeviceCode`.""" + `ActiveDirectoryInteractive`, `ActiveDirectoryMsi`, `ActiveDirectoryDefault` (alias + `default`), `ActiveDirectoryDeviceCode`.""" azure_tenant_id: str | None = None """Entra ID tenant id, used with `ActiveDirectoryServicePrincipal` authentication.""" @@ -274,10 +184,9 @@ def get_query(self) -> Dict[str, Any]: return query def on_partial(self) -> None: - setup_token_credential(self) if self.authentication: - # Entra ID methods (token or driver-native) supply their own credentials and do not - # rely on username/password; resolve once we have a target. `on_resolved` validates. + # Entra ID methods supply their own credentials and do not rely on username/password; + # resolve once we have a target. `on_resolved` validates. if self.host and self.database: self.resolve() elif not self.is_partial(): @@ -307,7 +216,7 @@ def to_odbc_dsn(self) -> str: return build_odbc_dsn(params) def to_odbc_attrs_before(self) -> dict[int, bytes] | None: - """Return `attrs_before` with an Entra ID access token, or None for driver-native auth.""" + """Return `attrs_before` with a directly injected Entra ID access token, or None.""" return build_token_attrs_before(self) diff --git a/dlt/destinations/impl/mssql/sql_client.py b/dlt/destinations/impl/mssql/sql_client.py index ad654d0ff0..ac3de90d54 100644 --- a/dlt/destinations/impl/mssql/sql_client.py +++ b/dlt/destinations/impl/mssql/sql_client.py @@ -37,8 +37,15 @@ def __init__( self.credentials = credentials def open_connection(self) -> mssql_python.Connection: - # mssql-python bundles its own driver, so the connection string carries no DRIVER. For - # Entra ID token authentication a fresh access token is injected via attrs_before. + # mssql-python bundles its own driver, so the connection string carries no DRIVER, and it + # signs in for every supported Entra ID authentication method itself from the + # `Authentication=` DSN keyword — dlt injects no `attrs_before` for those. + # + # mssql-python auto-enables connection pooling (default: 100 connections, 600s idle + # timeout) on the first connection any process opens, unless the application calls + # `mssql_python.pooling()` first — which dlt does not do, since these defaults are + # already sane for our workload. The pool matches purely on connection-string text, so + # our credentials building a stable DSN is what makes reuse actually happen. self._conn = mssql_python.connect( self.credentials.to_odbc_dsn(), autocommit=True, diff --git a/docs/website/docs/dlt-ecosystem/destinations/fabric.md b/docs/website/docs/dlt-ecosystem/destinations/fabric.md index ffda58b133..c728b0312a 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/fabric.md +++ b/docs/website/docs/dlt-ecosystem/destinations/fabric.md @@ -33,31 +33,19 @@ Fabric Warehouse authenticates with Microsoft Entra ID. Whichever method you cho - Select **SQL endpoint** - Copy the **SQL connection string** - it should be in the format: `.datawarehouse.fabric.microsoft.com` -The authentication method is selected with the `authentication` credential option. `dlt` supports -two families of methods. - -With the **driver-native** methods, the ODBC driver performs the Entra ID sign-in: +The authentication method is selected with the `authentication` credential option; `dlt` writes +it to the connection string as `Authentication=`, and the +[mssql-python](https://github.com/microsoft/mssql-python) driver performs the Entra ID sign-in. | `authentication` | Description | Required fields | |---|---|---| | `ActiveDirectoryServicePrincipal` (default) | Service Principal | `azure_tenant_id`, `azure_client_id`, `azure_client_secret` | | `ActiveDirectoryPassword` | Entra ID username/password | `username`, `password` | | `ActiveDirectoryIntegrated` | Integrated Windows authentication | None | -| `ActiveDirectoryInteractive` | Interactive browser prompt (driver) | None | -| `ActiveDirectoryMsi` | Managed identity (driver) | None | - -With the **azure-identity** methods, `dlt` acquires an access token with -[azure-identity](https://learn.microsoft.com/python/api/overview/azure/identity-readme) and injects -it into the connection. These work cross-platform (including macOS, where the ODBC driver's built-in -Entra ID modes are unreliable) and need no secret in `secrets.toml`: - -| `authentication` | azure-identity credential | -|---|---| -| `ActiveDirectoryDefault` (alias `default`) | `DefaultAzureCredential` (managed identity, environment, Azure CLI, …) | -| `ActiveDirectoryDeviceCode` | `DeviceCodeCredential` | - -When `authentication` is left at its default but no Service Principal secret is configured, `dlt` -falls back to `ActiveDirectoryDefault` (`DefaultAzureCredential`). +| `ActiveDirectoryInteractive` | Interactive browser prompt | None | +| `ActiveDirectoryMsi` | Managed identity | None | +| `ActiveDirectoryDefault` (alias `default`) | Managed identity, environment, Azure CLI, … (via `DefaultAzureCredential`) | None | +| `ActiveDirectoryDeviceCode` | Device code flow | None | ### Create a pipeline @@ -90,7 +78,7 @@ port = 1433 connect_timeout = 30 ``` -azure-identity, e.g. `DefaultAzureCredential` after `az login`, which needs no secret: +Passwordless, e.g. `DefaultAzureCredential` after `az login`: ```toml [destination.fabric.credentials] @@ -228,7 +216,7 @@ no ODBC driver name needs to be configured. While Fabric Warehouse is based on SQL Server, there are key differences: -1. **Authentication**: Fabric uses Entra ID; in addition to Service Principal, `dlt` supports several azure-identity methods (see [Authentication](#authentication)) +1. **Authentication**: Fabric uses Entra ID; in addition to Service Principal, `dlt` supports several other methods (see [Authentication](#authentication)) 2. **Type System**: Uses `varchar` and `datetime2` instead of `nvarchar` and `datetimeoffset` 3. **Collation**: Optimized for UTF-8 collations, with long/max types handled natively by the driver 4. **SQL Dialect**: Uses `fabric` SQLglot dialect for proper SQL generation diff --git a/docs/website/docs/dlt-ecosystem/destinations/mssql.md b/docs/website/docs/dlt-ecosystem/destinations/mssql.md index f096feb9af..c619ac7b15 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/mssql.md +++ b/docs/website/docs/dlt-ecosystem/destinations/mssql.md @@ -90,24 +90,22 @@ Long strings (>8k) are handled automatically by the driver, no extra configurati ### Microsoft Entra ID authentication For Azure-hosted SQL Server (Azure SQL Database, Managed Instance) you can authenticate with -Entra ID instead of a SQL login. Set the `authentication` credential option. +Entra ID instead of a SQL login. Set the `authentication` credential option to one of the methods +below; `dlt` writes it to the connection string as `Authentication=`, and the +[mssql-python](https://github.com/microsoft/mssql-python) driver performs the sign-in. -With the **azure-identity** methods, `dlt` acquires an access token and injects it into the -connection, so they work cross-platform (including macOS) and need no password in `secrets.toml`. -They require the `azure-identity` package (installed with `pip install "dlt[az]"`). - -| `authentication` | How it authenticates | +| `authentication` | Description | |---|---| | _(empty, default)_ | SQL login with `username`/`password` | -| `ActiveDirectoryServicePrincipal` | Service Principal (`azure_tenant_id`, `azure_client_id`, `azure_client_secret`), handled by the ODBC driver | -| `ActiveDirectoryPassword` | Entra ID `username`/`password` (handled by the ODBC driver) | -| `ActiveDirectoryIntegrated` | Integrated Windows authentication (handled by the ODBC driver) | -| `ActiveDirectoryInteractive` | Interactive prompt (handled by the ODBC driver) | -| `ActiveDirectoryMsi` | Managed identity (handled by the ODBC driver) | -| `ActiveDirectoryDefault` (alias `default`) | `DefaultAzureCredential` (managed identity, environment, Azure CLI, …), token injected by dlt | -| `ActiveDirectoryDeviceCode` | `DeviceCodeCredential`, token injected by dlt | - -Passwordless example using `DefaultAzureCredential` (e.g. after `az login`): +| `ActiveDirectoryServicePrincipal` | Service Principal (`azure_tenant_id`, `azure_client_id`, `azure_client_secret`) | +| `ActiveDirectoryPassword` | Entra ID `username`/`password` | +| `ActiveDirectoryIntegrated` | Integrated Windows authentication | +| `ActiveDirectoryInteractive` | Interactive browser prompt | +| `ActiveDirectoryMsi` | Managed identity | +| `ActiveDirectoryDefault` (alias `default`) | Managed identity, environment, Azure CLI, … (via `DefaultAzureCredential`) | +| `ActiveDirectoryDeviceCode` | Device code flow | + +Passwordless example (e.g. after `az login`): ```toml [destination.mssql.credentials] database = "dlt_data" @@ -126,9 +124,6 @@ azure_client_id = "your-client-id" azure_client_secret = "your-client-secret" ``` -When `authentication` is left empty but no `password` is set, `dlt` falls back to -`DefaultAzureCredential`. - **To pass credentials directly**, use the [explicit instance of the destination](../../general-usage/destination.md#pass-explicit-credentials) ```py pipeline = dlt.pipeline( diff --git a/docs/website/docs/dlt-ecosystem/destinations/synapse.md b/docs/website/docs/dlt-ecosystem/destinations/synapse.md index 3e0136c51f..d8751aaa26 100644 --- a/docs/website/docs/dlt-ecosystem/destinations/synapse.md +++ b/docs/website/docs/dlt-ecosystem/destinations/synapse.md @@ -97,8 +97,7 @@ pipeline = dlt.pipeline( ``` **Microsoft Entra ID authentication** is supported through the same `authentication` credential option as the [mssql destination](./mssql.md#microsoft-entra-id-authentication), including the -passwordless azure-identity methods (`ActiveDirectoryDefault`, `ActiveDirectoryDeviceCode`) that -inject an access token. +passwordless `ActiveDirectoryDefault` and `ActiveDirectoryDeviceCode` methods. To use **Active Directory Principal**, you can use the `sqlalchemy.engine.URL.create` method to create the connection URL using your Active Directory Service Principal credentials. First, create the connection string as: ```py diff --git a/tests/load/fabric/test_fabric_configuration.py b/tests/load/fabric/test_fabric_configuration.py index cd0ec26e5a..dff2475088 100644 --- a/tests/load/fabric/test_fabric_configuration.py +++ b/tests/load/fabric/test_fabric_configuration.py @@ -1,7 +1,6 @@ """Tests for Microsoft Fabric Warehouse destination configuration""" import os -import struct from typing import Optional import pytest @@ -15,7 +14,6 @@ FabricCredentials, FabricClientConfiguration, ) -from dlt.destinations.impl.mssql.configuration import uses_token_authentication # mark all tests as essential, do not remove pytestmark = pytest.mark.essential @@ -149,13 +147,11 @@ def test_fabric_type_mapper() -> None: def test_fabric_credentials_missing_service_principal() -> None: - """Test that Service Principal fields can trigger default credentials fallback""" + """Test that credentials can be built without Service Principal fields set""" creds = FabricCredentials() creds.host = "test.datawarehouse.fabric.microsoft.com" creds.database = "testdb" - # When Service Principal fields are missing, on_partial should attempt to use default credentials - # We can't test actual Azure default credentials in unit tests, but we can verify the structure assert creds.host == "test.datawarehouse.fabric.microsoft.com" assert creds.database == "testdb" @@ -226,17 +222,6 @@ def test_fabric_credentials_authentication_method() -> None: # --------------------------------------------------------------------------- -class _FakeAccessToken: - token = "fake-access-token" - - -class _FakeTokenCredential: - """Minimal azure-identity-like credential, avoids hitting Azure in unit tests.""" - - def get_token(self, *scopes: str, **kwargs: object) -> _FakeAccessToken: - return _FakeAccessToken() - - def _warehouse_credentials( authentication: str | None = None, **kwargs: object ) -> FabricCredentials: @@ -254,57 +239,49 @@ def test_fabric_authentication_default_is_service_principal() -> None: assert FabricCredentials().authentication == "ActiveDirectoryServicePrincipal" -@pytest.mark.parametrize( - "authentication,expected", - [ - ("default", "DefaultAzureCredential"), - ("ActiveDirectoryDefault", "DefaultAzureCredential"), - ("ActiveDirectoryDeviceCode", "DeviceCodeCredential"), - ], -) -def test_fabric_azure_identity_credential_mapping(authentication: str, expected: str) -> None: - """Each azure-identity method maps to the right credential and skips the DSN AUTHENTICATION.""" - creds = _warehouse_credentials(authentication) +def test_fabric_default_alias_normalizes_in_dsn() -> None: + """The `default` alias resolves to the canonical name mssql-python recognizes.""" + creds = _warehouse_credentials("default") creds.on_partial() - assert type(creds.default_credentials()).__name__ == expected - dsn = creds.get_odbc_dsn_dict() - assert "AUTHENTICATION" not in dsn + assert dsn["AUTHENTICATION"] == "ActiveDirectoryDefault" assert "UID" not in dsn assert "PWD" not in dsn - assert "LongAsMax" not in dsn + assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False @pytest.mark.parametrize( "authentication", ["auto", "cli", "environment", "interactive", "devicecode", "msi", "managedidentity"], ) -def test_fabric_removed_dlt_custom_alias_raises(authentication: str) -> None: - """The old dlt-custom lowercase aliases were replaced by native ODBC/azure-identity names.""" +def test_fabric_unsupported_alias_raises(authentication: str) -> None: + """Only the canonical `ActiveDirectory*` names (and the `default` alias) are supported.""" creds = _warehouse_credentials(authentication) with pytest.raises(ConfigurationException): creds.on_partial() # resolves (host+database present) -> on_resolved -> validate raises -def test_fabric_service_principal_without_secret_falls_back_to_token() -> None: - """Default method without a Service Principal secret injects a DefaultAzureCredential token.""" +def test_fabric_service_principal_without_secret_passes_through() -> None: + """No secret configured: dlt does not fall back to anything else, same as any other method.""" creds = _warehouse_credentials() creds.on_partial() - assert uses_token_authentication(creds) is True - assert type(creds.default_credentials()).__name__ == "DefaultAzureCredential" - assert "AUTHENTICATION" not in creds.get_odbc_dsn_dict() + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + assert "UID" not in dsn + assert "PWD" not in dsn + assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False -def test_fabric_service_principal_with_secret_is_driver_native() -> None: - """Default method with a Service Principal secret authenticates through the ODBC driver.""" +def test_fabric_service_principal_with_secret() -> None: creds = _warehouse_credentials( azure_tenant_id="t", azure_client_id="c", azure_client_secret="s" ) creds.on_partial() - assert uses_token_authentication(creds) is False dsn = creds.get_odbc_dsn_dict() assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" assert dsn["UID"] == "c@t" @@ -314,9 +291,19 @@ def test_fabric_service_principal_with_secret_is_driver_native() -> None: @pytest.mark.parametrize( "authentication", - ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", "ActiveDirectoryMsi"], + [ + "ActiveDirectoryIntegrated", + "ActiveDirectoryInteractive", + "ActiveDirectoryMsi", + "ActiveDirectoryDefault", + "ActiveDirectoryDeviceCode", + ], ) -def test_fabric_driver_native_passthrough(authentication: str) -> None: +def test_fabric_authentication_method_passthrough(authentication: str) -> None: + """Written straight to `Authentication=`; dlt builds no credential or attrs_before. + + mssql-python performs the sign-in for every supported method itself. + """ creds = _warehouse_credentials(authentication) creds.on_partial() @@ -325,6 +312,7 @@ def test_fabric_driver_native_passthrough(authentication: str) -> None: assert "UID" not in dsn assert "PWD" not in dsn assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False def test_fabric_active_directory_password() -> None: @@ -352,21 +340,35 @@ def test_fabric_unsupported_authentication_raises() -> None: creds.on_partial() -def test_fabric_to_odbc_attrs_before_token_struct() -> None: - """The injected token follows the SQL_COPT_SS_ACCESS_TOKEN struct layout.""" - creds = _warehouse_credentials("default") - creds._set_default_credentials(_FakeTokenCredential()) +def test_fabric_to_odbc_attrs_before_always_none() -> None: + """mssql-python signs in for every supported authentication method itself: dlt injects + nothing, regardless of what's configured.""" + creds = _warehouse_credentials("ActiveDirectoryDefault") + assert creds.to_odbc_attrs_before() is None + + creds = _warehouse_credentials("ActiveDirectoryServicePrincipal") # no secret set + assert creds.to_odbc_attrs_before() is None + + +def test_fabric_resolve_configuration_service_principal_without_secret() -> None: + """Resolution succeeds without a Service Principal secret; dlt does not fall back to + anything — the DSN just carries the method with no credentials attached.""" + creds = FabricCredentials() + creds.host = "abc.datawarehouse.fabric.microsoft.com" + creds.database = "mydb" + + resolved = resolve_configuration(creds) - attrs = creds.to_odbc_attrs_before() - assert attrs is not None - token_struct = attrs[1256] # SQL_COPT_SS_ACCESS_TOKEN - length = struct.unpack(" None: - """Resolution succeeds without a Service Principal secret for token authentication.""" +def test_fabric_resolve_configuration_authentication_passthrough() -> None: + """A full `resolve_configuration()` round-trip writes the method straight to the DSN.""" creds = FabricCredentials() creds.host = "abc.datawarehouse.fabric.microsoft.com" creds.database = "mydb" @@ -375,6 +377,5 @@ def test_fabric_resolve_configuration_token_authentication() -> None: resolved = resolve_configuration(creds) assert resolved.is_resolved() - assert uses_token_authentication(resolved) is True - assert type(resolved.default_credentials()).__name__ == "DeviceCodeCredential" - assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() + assert resolved.to_odbc_attrs_before() is None + assert resolved.get_odbc_dsn_dict()["AUTHENTICATION"] == "ActiveDirectoryDeviceCode" diff --git a/tests/load/mssql/test_mssql_configuration.py b/tests/load/mssql/test_mssql_configuration.py index 51b1da0c6e..ca078fff1b 100644 --- a/tests/load/mssql/test_mssql_configuration.py +++ b/tests/load/mssql/test_mssql_configuration.py @@ -1,5 +1,4 @@ import os -import struct import pytest @@ -11,7 +10,6 @@ from dlt.destinations.impl.mssql.configuration import ( MsSqlClientConfiguration, MsSqlCredentials, - uses_token_authentication, validate_authentication, ) @@ -170,17 +168,6 @@ def test_to_odbc_dsn_connect_timeout_and_longasmax_dropped() -> None: # --------------------------------------------------------------------------- -class _FakeAccessToken: - token = "fake-access-token" - - -class _FakeTokenCredential: - """Minimal azure-identity-like credential, avoids hitting Azure in unit tests.""" - - def get_token(self, *scopes: str, **kwargs: object) -> _FakeAccessToken: - return _FakeAccessToken() - - def _mssql_credentials(authentication: object = None, **kwargs: object) -> MsSqlCredentials: creds = MsSqlCredentials() creds.host = "sql.example.com" @@ -207,38 +194,35 @@ def test_mssql_sql_login_dsn_uses_uid_pwd() -> None: assert creds.to_odbc_attrs_before() is None -@pytest.mark.parametrize( - "authentication,expected", - [ - ("default", "DefaultAzureCredential"), - ("ActiveDirectoryDefault", "DefaultAzureCredential"), - ("ActiveDirectoryDeviceCode", "DeviceCodeCredential"), - ], -) -def test_mssql_azure_identity_credential_mapping(authentication: str, expected: str) -> None: - creds = _mssql_credentials(authentication) - creds.on_partial() +def test_mssql_default_alias_normalizes_in_dsn() -> None: + """The `default` alias resolves to the canonical name mssql-python recognizes. - assert type(creds.default_credentials()).__name__ == expected + mssql-python only understands `ActiveDirectoryDefault` in the `Authentication=` DSN keyword, + not the thin dlt-side alias, so this must be written to the DSN in its normalized form. + """ + creds = _mssql_credentials("default") + creds.on_partial() dsn = creds.get_odbc_dsn_dict() - assert "AUTHENTICATION" not in dsn + assert dsn["AUTHENTICATION"] == "ActiveDirectoryDefault" assert "UID" not in dsn assert "PWD" not in dsn + assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False @pytest.mark.parametrize( "authentication", ["auto", "cli", "environment", "interactive", "devicecode", "msi", "managedidentity"], ) -def test_mssql_removed_dlt_custom_alias_raises(authentication: str) -> None: - """The old dlt-custom lowercase aliases were replaced by native ODBC/azure-identity names.""" +def test_mssql_unsupported_alias_raises(authentication: str) -> None: + """Only the canonical `ActiveDirectory*` names (and the `default` alias) are supported.""" creds = _mssql_credentials(authentication) with pytest.raises(ConfigurationException): validate_authentication(creds) -def test_mssql_service_principal_driver_native() -> None: +def test_mssql_service_principal_with_secret() -> None: creds = _mssql_credentials( "ActiveDirectoryServicePrincipal", azure_tenant_id="t", @@ -247,7 +231,6 @@ def test_mssql_service_principal_driver_native() -> None: ) creds.on_partial() - assert uses_token_authentication(creds) is False dsn = creds.get_odbc_dsn_dict() assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" assert dsn["UID"] == "c@t" @@ -255,20 +238,34 @@ def test_mssql_service_principal_driver_native() -> None: assert creds.to_odbc_attrs_before() is None -def test_mssql_service_principal_without_secret_falls_back_to_token() -> None: +def test_mssql_service_principal_without_secret_passes_through() -> None: + """No secret configured: dlt does not fall back to anything else, same as any other method.""" creds = _mssql_credentials("ActiveDirectoryServicePrincipal") creds.on_partial() - assert uses_token_authentication(creds) is True - assert type(creds.default_credentials()).__name__ == "DefaultAzureCredential" - assert "AUTHENTICATION" not in creds.get_odbc_dsn_dict() + dsn = creds.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + assert "UID" not in dsn + assert "PWD" not in dsn + assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False @pytest.mark.parametrize( "authentication", - ["ActiveDirectoryIntegrated", "ActiveDirectoryInteractive", "ActiveDirectoryMsi"], + [ + "ActiveDirectoryIntegrated", + "ActiveDirectoryInteractive", + "ActiveDirectoryMsi", + "ActiveDirectoryDefault", + "ActiveDirectoryDeviceCode", + ], ) -def test_mssql_driver_native_passthrough(authentication: str) -> None: +def test_mssql_authentication_method_passthrough(authentication: str) -> None: + """Written straight to `Authentication=`; dlt builds no credential or attrs_before. + + mssql-python performs the sign-in for every supported method itself. + """ creds = _mssql_credentials(authentication) creds.on_partial() @@ -277,6 +274,7 @@ def test_mssql_driver_native_passthrough(authentication: str) -> None: assert "UID" not in dsn assert "PWD" not in dsn assert creds.to_odbc_attrs_before() is None + assert creds.has_default_credentials() is False def test_mssql_active_directory_password() -> None: @@ -303,19 +301,36 @@ def test_mssql_unsupported_authentication_raises() -> None: creds.on_partial() # resolves (all present) -> on_resolved -> validate raises -def test_mssql_to_odbc_attrs_before_token_struct() -> None: - creds = _mssql_credentials("default") - creds._set_default_credentials(_FakeTokenCredential()) +def test_mssql_to_odbc_attrs_before_always_none() -> None: + """mssql-python signs in for every supported authentication method itself: dlt injects + nothing, regardless of what's configured.""" + creds = _mssql_credentials("ActiveDirectoryDefault") + assert creds.to_odbc_attrs_before() is None - attrs = creds.to_odbc_attrs_before() - assert attrs is not None - token_struct = attrs[1256] # SQL_COPT_SS_ACCESS_TOKEN - length = struct.unpack(" None: + """Resolution succeeds without a Service Principal secret; dlt does not fall back to + anything — the DSN just carries the method with no credentials attached.""" + creds = MsSqlCredentials() + creds.host = "sql.example.com" + creds.database = "test_db" + creds.authentication = "ActiveDirectoryServicePrincipal" + + resolved = resolve_configuration(creds) + + assert resolved.is_resolved() + assert resolved.to_odbc_attrs_before() is None + dsn = resolved.get_odbc_dsn_dict() + assert dsn["AUTHENTICATION"] == "ActiveDirectoryServicePrincipal" + assert "UID" not in dsn + assert "PWD" not in dsn -def test_mssql_resolve_configuration_token_authentication() -> None: +def test_mssql_resolve_configuration_authentication_passthrough() -> None: + """A full `resolve_configuration()` round-trip writes the method straight to the DSN.""" creds = MsSqlCredentials() creds.host = "sql.example.com" creds.database = "test_db" @@ -324,6 +339,5 @@ def test_mssql_resolve_configuration_token_authentication() -> None: resolved = resolve_configuration(creds) assert resolved.is_resolved() - assert uses_token_authentication(resolved) is True - assert type(resolved.default_credentials()).__name__ == "DeviceCodeCredential" - assert "AUTHENTICATION" not in resolved.get_odbc_dsn_dict() + assert resolved.to_odbc_attrs_before() is None + assert resolved.get_odbc_dsn_dict()["AUTHENTICATION"] == "ActiveDirectoryDeviceCode"