From 8e13911845c80e236c14837d08aadd1fb3d7b120 Mon Sep 17 00:00:00 2001 From: Alex Caruso Date: Tue, 20 Jan 2026 13:26:58 -0500 Subject: [PATCH] support extracting all values as text, represented exactly as in source spreadsheet This is helpful for destinations where users wish to handle type casting themselves, for example via tools like dbt --- sources/google_sheets/__init__.py | 34 ++++++++++++++++++---- sources/google_sheets/helpers/api_calls.py | 23 +++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/sources/google_sheets/__init__.py b/sources/google_sheets/__init__.py index 5bdb623c1..428af8f12 100644 --- a/sources/google_sheets/__init__.py +++ b/sources/google_sheets/__init__.py @@ -27,6 +27,7 @@ def google_spreadsheet( get_sheets: bool = False, get_named_ranges: bool = True, max_api_retries: int = 5, + extract_all_as_text: bool = False, ) -> Iterable[DltResource]: """ The source for the dlt pipeline. It returns the following resources: @@ -44,6 +45,10 @@ def google_spreadsheet( get_named_ranges (bool, optional): If True, load all the named ranges inside the spreadsheet into the database. Defaults to True. max_api_retries (int, optional): Max number of retires to google sheets API. Actual behavior is internal to google client. + extract_all_as_text (bool, optional): If True, extract all cell values as their displayed formatted strings + (preserving date formatting, currency symbols, etc.) and load all columns as VARCHAR/TEXT + in the destination. Use this when you want to preserve the exact visual representation + from Google Sheets. Defaults to False. Yields: Iterable[DltResource]: List of dlt resources. @@ -65,14 +70,19 @@ def google_spreadsheet( all_range_names.update(named_ranges) # first we get all data for all the ranges (explicit or named) + # use FORMATTED_VALUE when extract_all_as_text=True to get displayed strings instead of typed values + value_render_option = ( + "FORMATTED_VALUE" if extract_all_as_text else "UNFORMATTED_VALUE" + ) all_range_data = api_calls.get_data_for_ranges( service=service, spreadsheet_id=spreadsheet_id, range_names=list(all_range_names), + value_render_option=value_render_option, + ) + assert len(all_range_names) == len(all_range_data), ( + "Google Sheets API must return values for all requested ranges" ) - assert len(all_range_names) == len( - all_range_data - ), "Google Sheets API must return values for all requested ranges" # get metadata for two first rows of each range # first should contain headers @@ -126,7 +136,7 @@ def google_spreadsheet( headers = get_range_headers(headers_metadata, name) if headers is None: # generate automatic headers and treat the first row as data - headers = [f"col_{idx+1}" for idx in range(len(headers_metadata))] + headers = [f"col_{idx + 1}" for idx in range(len(headers_metadata))] data_row_metadata = headers_metadata rows_data = values[0:] logger.warning( @@ -139,11 +149,23 @@ def google_spreadsheet( data_types = get_data_types(data_row_metadata) - yield dlt.resource( - process_range(rows_data, headers=headers, data_types=data_types), + # When extract_all_as_text=True, skip date type inference (values are already strings) + # and force all columns to text type in destination + effective_data_types = ( + [None] * len(headers) if extract_all_as_text else data_types + ) + + resource = dlt.resource( + process_range(rows_data, headers=headers, data_types=effective_data_types), name=name, write_disposition="replace", ) + + if extract_all_as_text: + # Apply text type hints to force VARCHAR/TEXT in destination (e.g., Snowflake) + resource.apply_hints(columns={h: {"data_type": "text"} for h in headers}) + + yield resource yield dlt.resource( metadata_table, write_disposition="merge", diff --git a/sources/google_sheets/helpers/api_calls.py b/sources/google_sheets/helpers/api_calls.py index 06e756482..37011f285 100644 --- a/sources/google_sheets/helpers/api_calls.py +++ b/sources/google_sheets/helpers/api_calls.py @@ -11,6 +11,7 @@ from .data_processing import ParsedRange, trim_range_top_left + try: from apiclient.discovery import build, Resource except ImportError: @@ -105,7 +106,10 @@ def get_known_range_names( @retry_deco def get_data_for_ranges( - service: Resource, spreadsheet_id: str, range_names: List[str] + service: Resource, + spreadsheet_id: str, + range_names: List[str], + value_render_option: str = "UNFORMATTED_VALUE", ) -> List[Tuple[str, ParsedRange, ParsedRange, List[List[Any]]]]: """ Calls Google Sheets API to get data in a batch. This is the most efficient way to get data for multiple ranges inside a spreadsheet. @@ -114,20 +118,29 @@ def get_data_for_ranges( service (Resource): Object to make API calls to Google Sheets. spreadsheet_id (str): The ID of the spreadsheet. range_names (List[str]): List of range names. + value_render_option (str): How values should be rendered by the API: + - "UNFORMATTED_VALUE": Returns typed values (numbers as numbers, dates as serial numbers) + - "FORMATTED_VALUE": Returns all values as their displayed formatted strings Returns: List[DictStrAny]: A list of ranges with data in the same order as `range_names` """ + # When using FORMATTED_VALUE, dates/times come as formatted strings + # When using UNFORMATTED_VALUE, we want serial numbers for date conversion + date_time_option = ( + "FORMATTED_STRING" + if value_render_option == "FORMATTED_VALUE" + else "SERIAL_NUMBER" + ) + range_batch_resp = ( service.spreadsheets() .values() .batchGet( spreadsheetId=spreadsheet_id, ranges=range_names, - # un formatted returns typed values - valueRenderOption="UNFORMATTED_VALUE", - # will return formatted dates as a serial number - dateTimeRenderOption="SERIAL_NUMBER", + valueRenderOption=value_render_option, + dateTimeRenderOption=date_time_option, ) .execute() )