diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ff489c..f2503a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 + - name: Install GDAL dependencies + run: sudo apt install -y libgdal-dev - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/docker/raster_loader/Dockerfile b/docker/raster_loader/Dockerfile index 34165e6..dbc995b 100644 --- a/docker/raster_loader/Dockerfile +++ b/docker/raster_loader/Dockerfile @@ -1,4 +1,4 @@ -FROM osgeo/gdal:3.2.0 +FROM osgeo/gdal:ubuntu-small-3.6.0 ENV HOMEAPP=/code ENV PATH=$PATH:$HOMEAPP/.local/bin @@ -18,6 +18,7 @@ RUN apt-get update \ postgresql-client \ wget \ python3-pip \ + libgdal-dev \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* # Using a non-privileged user to own our code diff --git a/raster_loader/cli/bigquery.py b/raster_loader/cli/bigquery.py index f504fa2..8451a84 100644 --- a/raster_loader/cli/bigquery.py +++ b/raster_loader/cli/bigquery.py @@ -4,8 +4,9 @@ import click from functools import wraps, partial -from raster_loader.utils import get_default_table_name +from raster_loader.lib.utils import get_default_table_name from raster_loader.io.bigquery import BigQueryConnection, AccessTokenCredentials +from raster_loader.lib.valuelabels import validate_band_valuelabels def catch_exception(func=None, *, handle=Exception): @@ -101,6 +102,28 @@ def bigquery(args=None): type=int, default=6, ) +@click.option( + "--band-valuelabels", + help="Custom data for valuelabels in JSON format, or 'None' to use the RAT if present. " + "i.e: '{: , : , ...}'. " + "Could repeat --band-valuelabels to specify multiple bands data. " + "They will be considered in the order they appear in the file. " + "Note that you can set any value to 'None' to use the RAT for the corresponding band. " + "Also see --rat-valuelabels-mode parameter.", + type=str, + default=[], + multiple=True, + callback=validate_band_valuelabels, +) +@click.option( + "--rat-valuelabels-mode", + help="The Raster Attribute Table (RAT) will be used for valuelabels if it's present. " + "If 'auto' (default), two columns will be chosen automatically for " + "Values and Labels based on their content. " + "If 'interactive', the user will be prompted to select the columns.", + type=click.Choice(["auto", "interactive"]), + default="auto", +) @catch_exception() def upload( file_path, @@ -119,6 +142,8 @@ def upload( exact_stats=False, basic_stats=False, compression_level=6, + band_valuelabels=[], + rat_valuelabels_mode="auto", ): from raster_loader.io.common import ( get_number_of_blocks, @@ -194,6 +219,8 @@ def upload( basic_stats=basic_stats, compress=compress, compression_level=compression_level, + band_valuelabels=band_valuelabels, + rat_valuelabels_mode=rat_valuelabels_mode, ) click.echo("Raster file uploaded to Google BigQuery") diff --git a/raster_loader/cli/databricks.py b/raster_loader/cli/databricks.py index c5767de..6e4c877 100644 --- a/raster_loader/cli/databricks.py +++ b/raster_loader/cli/databricks.py @@ -4,8 +4,9 @@ import click from functools import wraps, partial -from raster_loader.utils import get_default_table_name +from raster_loader.lib.utils import get_default_table_name from raster_loader.io.databricks import DatabricksConnection +from raster_loader.lib.valuelabels import validate_band_valuelabels def catch_exception(func=None, *, handle=Exception): @@ -115,6 +116,28 @@ def databricks(args=None): type=int, default=6, ) +@click.option( + "--band-valuelabels", + help="Custom data for valuelabels in JSON format, or 'None' to use the RAT if present. " + "i.e: '{: , : , ...}'. " + "Could repeat --band-valuelabels to specify multiple bands data. " + "They will be considered in the order they appear in the file. " + "Note that you can set any value to 'None' to use the RAT for the corresponding band. " + "Also see --rat-valuelabels-mode parameter.", + type=str, + default=[], + multiple=True, + callback=validate_band_valuelabels, +) +@click.option( + "--rat-valuelabels-mode", + help="The Raster Attribute Table (RAT) will be used for valuelabels if it's present. " + "If 'auto' (default), two columns will be chosen automatically for " + "Values and Labels based on their content. " + "If 'interactive', the user will be prompted to select the columns.", + type=click.Choice(["auto", "interactive"]), + default="auto", +) @catch_exception() def upload( server_hostname, @@ -136,6 +159,8 @@ def upload( exact_stats=False, basic_stats=False, compression_level=6, + band_valuelabels=[], + rat_valuelabels_mode="auto", ): from raster_loader.io.common import ( get_number_of_blocks, @@ -213,6 +238,8 @@ def upload( basic_stats=basic_stats, compress=compress, compression_level=compression_level, + band_valuelabels=band_valuelabels, + rat_valuelabels_mode=rat_valuelabels_mode, ) click.echo("Raster file uploaded to Databricks") diff --git a/raster_loader/cli/snowflake.py b/raster_loader/cli/snowflake.py index d0056eb..0ab294b 100644 --- a/raster_loader/cli/snowflake.py +++ b/raster_loader/cli/snowflake.py @@ -4,8 +4,9 @@ import click from functools import wraps, partial -from raster_loader.utils import get_default_table_name, check_private_key +from raster_loader.lib.utils import get_default_table_name, check_private_key from raster_loader.io.snowflake import SnowflakeConnection +from raster_loader.lib.valuelabels import validate_band_valuelabels def catch_exception(func=None, *, handle=Exception): @@ -123,6 +124,28 @@ def snowflake(args=None): type=int, default=6, ) +@click.option( + "--band-valuelabels", + help="Custom data for valuelabels in JSON format, or 'None' to use the RAT if present. " + "i.e: '{: , : , ...}'. " + "Could repeat --band-valuelabels to specify multiple bands data. " + "They will be considered in the order they appear in the file. " + "Note that you can set any value to 'None' to use the RAT for the corresponding band. " + "Also see --rat-valuelabels-mode parameter.", + type=str, + default=[], + multiple=True, + callback=validate_band_valuelabels, +) +@click.option( + "--rat-valuelabels-mode", + help="The Raster Attribute Table (RAT) will be used for valuelabels if it's present. " + "If 'auto' (default), two columns will be chosen automatically for " + "Values and Labels based on their content. " + "If 'interactive', the user will be prompted to select the columns.", + type=click.Choice(["auto", "interactive"]), + default="auto", +) @catch_exception() def upload( account, @@ -148,6 +171,8 @@ def upload( exact_stats=False, basic_stats=False, compression_level=6, + band_valuelabels=[], + rat_valuelabels_mode="auto", ): from raster_loader.io.common import ( get_number_of_blocks, @@ -255,6 +280,8 @@ def upload( basic_stats=basic_stats, compress=compress, compression_level=compression_level, + band_valuelabels=band_valuelabels, + rat_valuelabels_mode=rat_valuelabels_mode, ) click.echo("Raster file uploaded to Snowflake") diff --git a/raster_loader/io/bigquery.py b/raster_loader/io/bigquery.py index e22951d..8bfa032 100644 --- a/raster_loader/io/bigquery.py +++ b/raster_loader/io/bigquery.py @@ -6,8 +6,8 @@ from itertools import chain from raster_loader import __version__ -from raster_loader.errors import import_error_bigquery, IncompatibleRasterException -from raster_loader.utils import ask_yes_no_question, batched +from raster_loader.lib.errors import import_error_bigquery, IncompatibleRasterException +from raster_loader.lib.utils import ask_yes_no_question, batched from raster_loader.io.common import ( check_metadata_is_compatible, get_number_of_blocks, @@ -18,7 +18,7 @@ update_metadata, ) -from typing import Iterable, List, Tuple +from typing import Dict, Iterable, List, Literal, Tuple from functools import partial try: @@ -111,6 +111,8 @@ def upload_raster( basic_stats: bool = False, compress: bool = False, compression_level: int = 6, + band_valuelabels: List[Dict[int, str]] = [], + rat_valuelabels_mode: Literal["auto", "interactive"] = "auto", ): """Write a raster file to a BigQuery table.""" print("Loading raster file to BigQuery...") @@ -139,6 +141,8 @@ def upload_raster( exact_stats, basic_stats, compress=compress, + band_valuelabels=band_valuelabels, + rat_valuelabels_mode=rat_valuelabels_mode, ) overviews_records_gen = rasterio_overview_to_records( diff --git a/raster_loader/io/common.py b/raster_loader/io/common.py index b9e2bb1..7d97116 100644 --- a/raster_loader/io/common.py +++ b/raster_loader/io/common.py @@ -7,19 +7,21 @@ from raster_loader._version import __version__ from collections import Counter -from typing import Dict, Callable, Iterable, List, Tuple, Union +from typing import Dict, Callable, Iterable, List, Literal, Tuple, Union from affine import Affine from shapely import wkt # Can not use directly from shapely.wkt +from raster_loader.lib.valuelabels import get_band_valuelabels import rio_cogeo import rasterio import quadbin -from raster_loader.geo import raster_bounds -from raster_loader.errors import ( +from raster_loader.lib.geo import raster_bounds +from raster_loader.lib.errors import ( error_not_google_compatible, ) -from raster_loader.utils import warnings +from raster_loader.lib.utils import warnings + DEFAULT_COG_BLOCK_SIZE = 256 @@ -210,6 +212,8 @@ def rasterio_metadata( exact_stats: bool = False, basic_stats: bool = False, compress: bool = False, + band_valuelabels: List[Dict[int, str]] = [], + rat_valuelabels_mode: Literal["auto", "interactive"] = "auto", ): """Open a raster file with rasterio.""" raster_info = rio_cogeo.cog_info(file_path).dict() @@ -278,12 +282,16 @@ def rasterio_metadata( band_nodata = band_value_as_string( raster_dataset, band, band_nodata_value(raster_dataset, band) ) + meta = { "band": band, "type": raster_band_type(raster_dataset, band), "name": band_field_name(band_name, band, band_rename_function), "colorinterp": band_colorinterp, "colortable": get_color_table(raster_dataset, band), + "valuelabels": get_band_valuelabels( + file_path, band, band_valuelabels, rat_valuelabels_mode + ), "stats": stats, "nodata": band_nodata, } @@ -311,6 +319,7 @@ def rasterio_metadata( "colorinterp": e["colorinterp"], "nodata": e["nodata"], "colortable": e["colortable"], + "valuelabels": e["valuelabels"], } for e in bands_metadata ] @@ -506,7 +515,7 @@ def compute_quantiles(data: List[Union[int, float]], cast_function: Callable) -> def get_stats( raster_dataset: rasterio.io.DatasetReader, band: int -) -> rasterio.Statistics: +) -> rasterio._io.Statistics: """Get statistics for a raster band.""" try: # stats method is supported since rasterio 1.4.0 and statistics @@ -937,7 +946,8 @@ def is_valid_raster_dataset(raster_dataset: rasterio.io.DatasetReader) -> bool: def band_without_stats(band): return { k: band[k] - for k in set(list(band.keys())) - set(["stats", "colorinterp", "colortable"]) + for k in set(list(band.keys())) + - set(["stats", "colorinterp", "colortable", "valuelabels"]) } diff --git a/raster_loader/io/databricks.py b/raster_loader/io/databricks.py index 7164dea..ee4c1a7 100644 --- a/raster_loader/io/databricks.py +++ b/raster_loader/io/databricks.py @@ -3,8 +3,11 @@ import rasterio from itertools import chain -from raster_loader.errors import import_error_databricks, IncompatibleRasterException -from raster_loader.utils import ask_yes_no_question, batched +from raster_loader.lib.errors import ( + import_error_databricks, + IncompatibleRasterException, +) +from raster_loader.lib.utils import ask_yes_no_question, batched from raster_loader.io.common import ( check_metadata_is_compatible, get_number_of_blocks, @@ -15,7 +18,7 @@ update_metadata, ) -from typing import Iterable, List, Tuple +from typing import Dict, Iterable, List, Literal, Tuple try: from databricks.connect import DatabricksSession @@ -141,6 +144,8 @@ def upload_raster( basic_stats: bool = False, compress: bool = False, compression_level: int = 6, + band_valuelabels: List[Dict[int, str]] = [], + rat_valuelabels_mode: Literal["auto", "interactive"] = "auto", ): """Write a raster file to a Databricks table.""" # Wait for cluster to be ready before starting the upload @@ -174,6 +179,8 @@ def upload_raster( exact_stats, basic_stats, compress=compress, + band_valuelabels=band_valuelabels, + rat_valuelabels_mode=rat_valuelabels_mode, ) overviews_records_gen = rasterio_overview_to_records( diff --git a/raster_loader/io/snowflake.py b/raster_loader/io/snowflake.py index 6ecd9c1..893321a 100644 --- a/raster_loader/io/snowflake.py +++ b/raster_loader/io/snowflake.py @@ -3,14 +3,14 @@ import pandas as pd from itertools import chain -from typing import Iterable, List, Tuple +from typing import Dict, Iterable, List, Literal, Tuple -from raster_loader.errors import ( +from raster_loader.lib.errors import ( IncompatibleRasterException, import_error_snowflake, ) -from raster_loader.utils import ask_yes_no_question, batched +from raster_loader.lib.utils import ask_yes_no_question, batched from raster_loader.io.common import ( rasterio_metadata, @@ -208,6 +208,8 @@ def upload_raster( basic_stats: bool = False, compress: bool = False, compression_level: int = 6, + band_valuelabels: List[Dict[int, str]] = [], + rat_valuelabels_mode: Literal["auto", "interactive"] = "auto", ) -> bool: """Write a raster file to a Snowflake table.""" @@ -243,6 +245,8 @@ def band_rename_function(x): exact_stats, basic_stats, compress, + band_valuelabels, + rat_valuelabels_mode, ) overviews_records_gen = rasterio_overview_to_records( diff --git a/raster_loader/lib/__init__.py b/raster_loader/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/raster_loader/errors.py b/raster_loader/lib/errors.py similarity index 100% rename from raster_loader/errors.py rename to raster_loader/lib/errors.py diff --git a/raster_loader/geo.py b/raster_loader/lib/geo.py similarity index 100% rename from raster_loader/geo.py rename to raster_loader/lib/geo.py diff --git a/raster_loader/utils.py b/raster_loader/lib/utils.py similarity index 100% rename from raster_loader/utils.py rename to raster_loader/lib/utils.py diff --git a/raster_loader/lib/valuelabels.py b/raster_loader/lib/valuelabels.py new file mode 100644 index 0000000..61c6fe0 --- /dev/null +++ b/raster_loader/lib/valuelabels.py @@ -0,0 +1,178 @@ +import json +from typing import Literal +import click +from osgeo import gdal + +DEFAULT_VALUES_COLUMN_IDX = 0 +DEFAULT_LABELS_COLUMN_IDX = 1 + +gdal.UseExceptions() + + +def get_band_valuelabels( + file_path: str, + band: int, + band_valuelabels: list[dict[int, str]], + rat_valuelabels_mode: Literal["auto", "interactive"], +) -> dict[int, str]: + if len(band_valuelabels) >= band and band_valuelabels[band - 1] is not None: + print(f"Using the provided valuelabels for band {band}") + valuelabels = band_valuelabels[band - 1] + else: + # Computing valuelabels from the Raster Attribute Table (RAT) + valuelabels = get_valuelabels_from_rat(file_path, band, rat_valuelabels_mode) + return valuelabels + + +def get_valuelabels_from_rat( + dataset_uri: str, band_idx: int, mode: Literal["auto", "interactive"] +) -> dict[int, str]: + """ + Get the value labels for a given dataset and band. + """ + print(f"Computing value labels for band {band_idx}") + try: + dataset = gdal.Open(dataset_uri) # dataset_uri is path to .tif file + band = dataset.GetRasterBand(band_idx) + + rat = band.GetDefaultRAT() + if rat is None: + print(f"No Raster Attribute Table (RAT) found for band {band_idx}") + return None + + print(f"Available columns in Raster Attribute Table (RAT) for band {band_idx}:") + for col_idx in range(rat.GetColumnCount()): + print(f"\t{col_idx}: {rat.GetNameOfCol(col_idx)}") + + if mode == "interactive": + values_column_idx = click.prompt( + f"Introduce the column index for Values for band {band_idx}", + type=int, + default=DEFAULT_VALUES_COLUMN_IDX, + ) + labels_column_idx = click.prompt( + f"Introduce the column index for Labels for band {band_idx}", + type=int, + default=DEFAULT_LABELS_COLUMN_IDX, + ) + else: + # Guess columns for values and labels + values_column_idx = get_values_column_idx(rat) + labels_column_idx = get_labels_column_idx(rat, [values_column_idx]) + + print( + f"Selected RAT column for Values: " + f"[{values_column_idx}: {rat.GetNameOfCol(values_column_idx)}]" + ) + print( + f"Selected RAT column for Labels: " + f"[{labels_column_idx}: {rat.GetNameOfCol(labels_column_idx)}]" + ) + + # Convert RAT to dictionary + value_labels = {} + for i in range(rat.GetRowCount()): + value = rat.GetValueAsInt(i, values_column_idx) + label = rat.GetValueAsString(i, labels_column_idx) + value_labels[value] = label + return value_labels + except ValueError: + return None + + +def get_values_column_idx(rat: gdal.RasterAttributeTable) -> int: + values_column_idx = DEFAULT_VALUES_COLUMN_IDX + for col_idx in range(rat.GetColumnCount()): + col_name = rat.GetNameOfCol(col_idx) + if col_name.lower() == "value": + values_column_idx = col_idx + + return values_column_idx + + +def get_labels_column_idx( + rat: gdal.RasterAttributeTable, skip_columns: list[int] = [] +) -> int: + """ + Get the column index of the labels column in a Raster Attribute Table (RAT). + + This function uses a heuristic approach to identify the most likely column + containing descriptive labels by: + 1. Excluding columns specified in skip_columns (typically the values column) + 2. Ranking remaining columns based on: + - Number of unique values in the column + - Number of unique words in the column + 3. Selecting the column with the highest combined score (unique values * unique words) + If no column has a score > 0, falls back to selecting the column with most unique values + + Args: + rat: GDAL Raster Attribute Table + skip_columns: List of column indices to exclude from consideration + + Returns: + int: The index of the column most likely to contain labels + """ + labels_column_idx = DEFAULT_LABELS_COLUMN_IDX + unique_values_count = count_column_unique_values(rat) + + unique_words_count = count_column_unique_words(rat) + + # Ranked columns by (unique values count * bigrams count) + ranked_columns = {} + for col_idx in range(rat.GetColumnCount()): + if col_idx in skip_columns: + continue + ranked_columns[col_idx] = ( + unique_values_count[col_idx] * unique_words_count[col_idx] + ) + + # Sort the ranked columns by the rank + ranked_columns = sorted(ranked_columns.items(), key=lambda x: x[1], reverse=True) + + # Get the column with the highest rank + if ranked_columns[0][1] > 0: + labels_column_idx = ranked_columns[0][0] + else: + # If no bigrams found, select column with most unique values + labels_column_idx = max(unique_values_count.items(), key=lambda x: x[1])[0] + + return labels_column_idx + + +# Return a dictionary with the column_idx and the number of unique values in it +def count_column_unique_values(rat: gdal.RasterAttributeTable) -> dict[int, int]: + unique_values_count = {} + for col_idx in range(rat.GetColumnCount()): + col_unique_values = set() + for row_idx in range(rat.GetRowCount()): + value = rat.GetValueAsString(row_idx, col_idx) + col_unique_values.add(value) + unique_values_count[col_idx] = len(col_unique_values) + + return unique_values_count + + +def count_column_unique_words(rat: gdal.RasterAttributeTable) -> dict[int, int]: + columns_words_count = {} + for col_idx in range(rat.GetColumnCount()): + column_text = "" + for row_idx in range(rat.GetRowCount()): + column_text += rat.GetValueAsString(row_idx, col_idx).lower() + " " + words_list = set(column_text.split()) + # Remove words which are not text + words_list = [w for w in words_list if w.isalpha()] + columns_words_count[col_idx] = len(words_list) + + return columns_words_count + + +def validate_band_valuelabels(_, __, value): + """ + Validate the band valuelabels parameter for click library callback. + """ + try: + return [json.loads(item) if item != "None" else None for item in value] + except json.JSONDecodeError: + raise click.BadParameter( + "Invalid JSON format. Please provide a valid JSON object." + ) diff --git a/raster_loader/tests/bigquery/test_cli.py b/raster_loader/tests/bigquery/test_cli.py index 5478e94..bb5d5b8 100644 --- a/raster_loader/tests/bigquery/test_cli.py +++ b/raster_loader/tests/bigquery/test_cli.py @@ -308,6 +308,33 @@ def test_bigquery_describe(*args, **kwargs): assert result.exit_code == 0 +@patch( + "raster_loader.cli.bigquery.BigQueryConnection.upload_records", return_value=None +) +@patch("raster_loader.cli.bigquery.BigQueryConnection.__init__", return_value=None) +def test_bigquery_upload_custom_valuelabels(*args, **kwargs): + runner = CliRunner() + result = runner.invoke( + main, + [ + "bigquery", + "upload", + "--file_path", + f"{tiff}", + "--project", + "project", + "--dataset", + "dataset", + "--table", + "table", + "--band-valuelabels", + '{"90": "Woody Wetlands", "52": "Shrub/Scrub"}', + ], + ) + assert result.exit_code == 1 + assert "Using the provided valuelabels for band 1" in result.output + + def test_info(*args, **kwargs): runner = CliRunner() result = runner.invoke(main, ["info"]) diff --git a/raster_loader/tests/bigquery/test_io.py b/raster_loader/tests/bigquery/test_io.py index f7a9e45..b982b76 100644 --- a/raster_loader/tests/bigquery/test_io.py +++ b/raster_loader/tests/bigquery/test_io.py @@ -483,6 +483,7 @@ def test_rasterio_to_table_overwrite(*args, **kwargs): }, "nodata": "0", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -649,6 +650,7 @@ def test_rasterio_to_table_invalid_raster(*args, **kwargs): }, "nodata": "0", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -744,6 +746,7 @@ def test_get_labels(*args, **kwargs): }, "nodata": "0", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -802,6 +805,7 @@ def test_rasterio_to_bigquery_with_compression(*args, **kwargs): }, "nodata": "0", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, diff --git a/raster_loader/tests/databricks/test_io.py b/raster_loader/tests/databricks/test_io.py index 1991bdb..82c34f7 100644 --- a/raster_loader/tests/databricks/test_io.py +++ b/raster_loader/tests/databricks/test_io.py @@ -391,6 +391,7 @@ def test_rasterio_to_table(*args, **kwargs): "nodata": "0", "colorinterp": "red", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, diff --git a/raster_loader/tests/snowflake/test_io.py b/raster_loader/tests/snowflake/test_io.py index d38c9e4..0437b7d 100644 --- a/raster_loader/tests/snowflake/test_io.py +++ b/raster_loader/tests/snowflake/test_io.py @@ -462,6 +462,7 @@ def test_rasterio_to_table_overwrite(*args, **kwargs): "nodata": "0", "colorinterp": "red", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -629,6 +630,7 @@ def test_rasterio_to_table_invalid_raster(*args, **kwargs): "nodata": "0", "colorinterp": "red", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -712,6 +714,7 @@ def test_append_with_different_resolution(*args, **kwargs): "nodata": "0", "colorinterp": "red", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, @@ -772,6 +775,7 @@ def test_rasterio_to_snowflake_with_compression(*args, **kwargs): "nodata": "0", "colorinterp": "red", "colortable": None, + "valuelabels": None, } ], "num_blocks": 1, diff --git a/raster_loader/tests/test_utils.py b/raster_loader/tests/test_utils.py index 24dfdde..5b1e892 100644 --- a/raster_loader/tests/test_utils.py +++ b/raster_loader/tests/test_utils.py @@ -1,7 +1,7 @@ import builtins from unittest.mock import patch -from raster_loader.utils import ask_yes_no_question +from raster_loader.lib.utils import ask_yes_no_question def test_ask_yes_no_question_answer_yes(): diff --git a/setup.cfg b/setup.cfg index f0c293b..5b0bcb2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,13 +34,14 @@ install_requires = click>=8.1.3 db-dtypes>=1.0.5 pandas>=1.3.4 - pyarrow>=10.0.1 + pyarrow>=10.0.1,<17.0.0 pyproj>=3.2.1 rasterio>=1.3a3 rio-cogeo>=3.5.0 shapely>=1.7.1 quadbin>=0.2.0 tqdm>=4.64.1 + GDAL==3.6.0 zip_safe = False [options.entry_points] @@ -69,6 +70,6 @@ all = %(databricks)s [flake8] -max-line-length = 88 +max-line-length = 100 ignore = E203 W503