Skip to content

fix: common mechanism for retries in insert, upsert and delete #347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
konstantinoscs merged 1 commit into from
Aug 18, 2026
+366 −79
Merged

fix: common mechanism for retries in insert, upsert and delete #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/test-python.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,7 @@ jobs:
PY

- name: Run Iceberg compatibility tests
run: uv run --no-sync pytest tests/tower/test_tables.py tests/tower/test_storage.py
run: >-
uv run --no-sync pytest
tests/tower/test_table*.py
tests/tower/test_storage.py
156 changes: 78 additions & 78 deletions src/tower/_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
from __future__ import annotations

import math
import os
import random
import time
from dataclasses import dataclass
from typing import List, Optional, TypeVar, Union
from typing import Callable, List, Optional, TypeVar, Union

from pyiceberg.exceptions import CommitFailedException, NoSuchTableError

TTable = TypeVar("TTable", bound="Table")

import random
import time
TRetryResult = TypeVar("TRetryResult")

import polars as pl
import pyarrow as pa
Expand Down Expand Up @@ -37,6 +38,8 @@
namespace_or_default,
)

_MAX_COMMIT_RETRY_DELAY_SECONDS = 30.0


@dataclass
class RowsAffectedInformation:
Expand Down Expand Up @@ -249,8 +252,34 @@ def rows_affected(self) -> RowsAffectedInformation:
def _validate_retry_args(max_retries: int, retry_delay_seconds: float) -> None:
if max_retries < 0:
raise ValueError("max_retries must be >= 0")
if retry_delay_seconds < 0:
raise ValueError("retry_delay_seconds must be >= 0")
if not math.isfinite(retry_delay_seconds) or retry_delay_seconds < 0:
raise ValueError("retry_delay_seconds must be finite and >= 0")
Comment on lines +255 to +256

@jo-sm jo-sm Aug 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest to give an upper bound (maybe 300 seconds / 5 minutes?), rather than only checking if the delay is finite.

@konstantinoscs konstantinoscs Aug 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a bit confusing because the upper bound is anyway decided in retry_ceiling = min(retry_delay_seconds, _MAX_COMMIT_RETRY_DELAY_SECONDS) inside _commit_with-retry_().
I'll add some clarification in the documentation for this

@jo-sm jo-sm Aug 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's handled somewhere else, I would just be consistent with the above line then - retry_delay_seconds < 0 is probably enough here.


def _commit_with_retry(
self,
operation: Callable[[], TRetryResult],
max_retries: int,
initial_retry_ceiling_seconds: float,
) -> TRetryResult:
retry_ceiling_seconds = min(
initial_retry_ceiling_seconds, _MAX_COMMIT_RETRY_DELAY_SECONDS
)

for attempt in range(max_retries + 1):
try:
return operation()
except CommitFailedException:
if attempt == max_retries:
raise

@jo-sm jo-sm Aug 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would raise a new exception, like CommitRetryExhaustionException.

@konstantinoscs konstantinoscs Aug 14, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should remain CommitFailedException because it's actually the PyIceberg exception.
Preserving that exception:

  1. retains the original failure details;
  2. keeps existing except CommitFailedException handlers working;
  3. avoids introducing Tower-specific behavior for S3 Tables and BYO callers.

A new exception would be a breaking change unless it subclassed CommitFailedException, and even then it adds little value.


delay_seconds = random.uniform(0.0, retry_ceiling_seconds)
time.sleep(delay_seconds)
self._table.refresh()
retry_ceiling_seconds = min(
retry_ceiling_seconds * 2, _MAX_COMMIT_RETRY_DELAY_SECONDS
)

raise AssertionError("unreachable")

def insert(
self,
Expand All @@ -270,8 +299,9 @@ def insert(
must match the schema of the target table.
max_retries (int): Maximum number of retry attempts on commit conflicts.
Defaults to 5.
retry_delay_seconds (float): Wait time in seconds between retries.
Defaults to 0.5 seconds.
retry_delay_seconds (float): Maximum randomized wait before the first retry,
in seconds. The maximum doubles after each conflict but never exceeds
30 seconds; values above 30 are treated as 30. Defaults to 0.5 seconds.

Returns:
TTable: The table instance with the newly inserted rows, allowing for method chaining.
Expand All @@ -296,23 +326,11 @@ def insert(
self._validate_retry_args(max_retries, retry_delay_seconds)
self._ensure_read_write_table()

last_exception = None

for attempt in range(max_retries + 1):
try:
if attempt > 0:
self._table.refresh()

self._table.append(data)
self._stats.inserts += data.num_rows
return self

except CommitFailedException as e:
last_exception = e
if attempt < max_retries:
time.sleep(retry_delay_seconds)

raise last_exception
self._commit_with_retry(
lambda: self._table.append(data), max_retries, retry_delay_seconds
)
self._stats.inserts += data.num_rows
return self

def upsert(
self,
Expand All @@ -337,8 +355,9 @@ def upsert(
If not provided, all columns will be used for matching.
max_retries (int): Maximum number of retry attempts on commit conflicts.
Defaults to 5.
retry_delay_seconds (float): Wait time in seconds between retries.
Defaults to 0.5 seconds.
retry_delay_seconds (float): Maximum randomized wait before the first retry,
in seconds. The maximum doubles after each conflict but never exceeds
30 seconds; values above 30 are treated as 30. Defaults to 0.5 seconds.

Returns:
TTable: The table instance with the upserted rows, allowing for method chaining.
Expand Down Expand Up @@ -370,34 +389,24 @@ def upsert(
self._validate_retry_args(max_retries, retry_delay_seconds)
self._ensure_read_write_table()

last_exception = None

for attempt in range(max_retries + 1):
try:
if attempt > 0:
self._table.refresh()

res = self._table.upsert(
data,
join_cols=join_cols,
# All upserts will always be case sensitive. Perhaps we'll add this
# as a parameter in the future?
case_sensitive=True,
# These are the defaults, but we're including them to be complete.
when_matched_update_all=True,
when_not_matched_insert_all=True,
)

self._stats.updates += res.rows_updated
self._stats.inserts += res.rows_inserted
return self

except CommitFailedException as e:
last_exception = e
if attempt < max_retries:
time.sleep(retry_delay_seconds)
res = self._commit_with_retry(
lambda: self._table.upsert(
data,
join_cols=join_cols,
# All upserts will always be case sensitive. Perhaps we'll add this
# as a parameter in the future?
case_sensitive=True,
# These are the defaults, but we're including them to be complete.
when_matched_update_all=True,
when_not_matched_insert_all=True,
),
max_retries,
retry_delay_seconds,
)

raise last_exception
self._stats.updates += res.rows_updated
self._stats.inserts += res.rows_inserted
return self

def delete(
self,
Expand All @@ -421,8 +430,9 @@ def delete(
- A string expression
max_retries (int): Maximum number of retry attempts on commit conflicts.
Defaults to 5.
retry_delay_seconds (float): Wait time in seconds between retries.
Defaults to 0.5 seconds.
retry_delay_seconds (float): Maximum randomized wait before the first retry,
in seconds. The maximum doubles after each conflict but never exceeds
30 seconds; values above 30 are treated as 30. Defaults to 0.5 seconds.

Returns:
TTable: The table instance with the deleted rows, allowing for method chaining.
Expand Down Expand Up @@ -455,30 +465,20 @@ def delete(
next_filters = convert_pyarrow_expressions(filters)
filters = next_filters

last_exception = None

for attempt in range(max_retries + 1):
try:
if attempt > 0:
self._table.refresh()

self._table.delete(
delete_filter=filters,
# We want this to always be the case. Not sure why you wouldn't?
case_sensitive=True,
)

# NOTE: There is, unfortunately, no way to get the number of rows
# deleted besides comparing the two snapshots that were created.

return self
self._commit_with_retry(
lambda: self._table.delete(
delete_filter=filters,
# We want this to always be the case. Not sure why you wouldn't?
case_sensitive=True,
),
max_retries,
retry_delay_seconds,
)

except CommitFailedException as e:
last_exception = e
if attempt < max_retries:
time.sleep(retry_delay_seconds)
# NOTE: There is, unfortunately, no way to get the number of rows
# deleted besides comparing the two snapshots that were created.

raise last_exception
return self

def schema(self) -> pa.Schema:
"""
Expand Down
Loading
Loading