feat: add refresh options for on-demand dataset refresh

README.md

@@ -140,6 +140,43 @@ using var client = new SpiceClientBuilder().Build();
 await client.RefreshDatasetAsync("my_dataset");
 ```
 
+##### Refresh Options
+
+Override the dataset's configured refresh settings for a single refresh by passing
+`RefreshOptions`. Any option left unset falls back to the dataset's Spicepod configuration.
+
+```csharp
+using Spice;
+using Spice.Datasets;
+
+using var client = new SpiceClientBuilder().Build();
+
+await client.RefreshDatasetAsync("taxi_trips", new RefreshOptions()
+    .WithRefreshSql("SELECT * FROM taxi_trips WHERE tip_amount > 10.0")
+    .WithRefreshMode(RefreshMode.Append)
+    .WithMaxJitter(TimeSpan.FromSeconds(10)));
+```
+
+Object initializer syntax works too:
+
+```csharp
+await client.RefreshDatasetAsync("taxi_trips", new RefreshOptions
+{
+    RefreshSql = "SELECT * FROM taxi_trips WHERE tip_amount > 10.0",
+    RefreshMode = RefreshMode.Append,
+    MaxJitter = TimeSpan.FromSeconds(10),
+});
+```
+
+| Option | Type | Description |
+| --- | --- | --- |
+| `RefreshSql` | `string` | The SQL statement used for this refresh. Defaults to the dataset's `refresh_sql`. |
+| `RefreshMode` | `RefreshMode` | `Full` replaces the accelerated data; `Append` adds newly returned rows. Defaults to the dataset's `refresh_mode`. |
+| `MaxJitter` | `TimeSpan` | Maximum jitter added before the refresh starts. Defaults to the dataset's `refresh_jitter_max`. |
+
+> **Note**: On-demand refreshes apply to the `full` and `append` refresh modes. Datasets accelerated with
+> `changes` mode are kept up to date by change data capture and are not refreshed through this API.
+
 #### Custom Connection Settings
 
 ```csharp
@@ -207,12 +244,17 @@ var data = await client.Query(
 
 ```csharp
 using Spice;
+using Spice.Datasets;
 
 using var client = new SpiceClientBuilder()
     .WithSpiceCloud("API_KEY")
     .Build();
 
 await client.RefreshDatasetAsync("my_dataset");
+
+// Or with refresh overrides for this refresh only
+await client.RefreshDatasetAsync("my_dataset", new RefreshOptions()
+    .WithRefreshMode(RefreshMode.Append));
 ```
 
 ### Memory Management

Spice/Spice.csproj

@@ -55,4 +55,9 @@
         </AssemblyAttribute>
     </ItemGroup>
 
+    <!-- System.Text.Json ships in-box for .NET 8.0+; netstandard2.0 needs the package. -->
+    <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
+        <PackageReference Include="System.Text.Json" Version="9.0.9" />
+    </ItemGroup>
+
 </Project>

Spice/src/Datasets/RefreshOptions.cs

@@ -0,0 +1,179 @@
+/*
+Copyright 2024 The Spice.ai OSS Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+using System.Globalization;
+using System.Text.Json;
+
+namespace Spice.Datasets;
+
+/// <summary>
+/// The refresh mode to use for a single on-demand dataset refresh.
+/// </summary>
+/// <remarks>
+/// On-demand refreshes apply to the <c>full</c> and <c>append</c> refresh modes only.
+/// Datasets accelerated with <c>changes</c> mode are kept up to date by change data
+/// capture and are not refreshed through this API.
+/// </remarks>
+public enum RefreshMode
+{
+    /// <summary>
+    /// Replace the accelerated data with the full result of the refresh query.
+    /// </summary>
+    Full,
+
+    /// <summary>
+    /// Append newly returned rows to the accelerated data.
+    /// </summary>
+    Append,
+}
+
+/// <summary>
+/// Optional overrides for a single on-demand dataset acceleration refresh.
+/// </summary>
+/// <remarks>
+/// Every option is optional. Any option left unset falls back to the value configured
+/// for the dataset in the Spicepod. An instance with no options set requests a refresh
+/// using the dataset's existing configuration.
+/// </remarks>
+/// <example>
+/// <code>
+/// var options = new RefreshOptions()
+///     .WithRefreshSql("SELECT * FROM taxi_trips WHERE tip_amount &gt; 10.0")
+///     .WithRefreshMode(RefreshMode.Append)
+///     .WithMaxJitter(TimeSpan.FromSeconds(10));
+///
+/// await client.RefreshDatasetAsync("taxi_trips", options);
+/// </code>
+/// </example>
+public sealed class RefreshOptions
+{
+    /// <summary>
+    /// The SQL statement used for this refresh. Defaults to the <c>refresh_sql</c>
+    /// configured for the dataset, if any.
+    /// </summary>
+    public string? RefreshSql { get; set; }
+
+    /// <summary>
+    /// The refresh mode to use for this refresh. Defaults to the <c>refresh_mode</c>
+    /// configured for the dataset, or <see cref="Datasets.RefreshMode.Full"/>.
+    /// </summary>
+    public RefreshMode? RefreshMode { get; set; }
+
+    /// <summary>
+    /// The maximum amount of jitter to add before starting this refresh. Defaults to the
+    /// <c>refresh_jitter_max</c> configured for the dataset, or 10% of the refresh check interval.
+    /// </summary>
+    public TimeSpan? MaxJitter { get; set; }
+
+    /// <summary>
+    /// Sets the SQL statement used for this refresh.
+    /// </summary>
+    /// <param name="refreshSql">The refresh SQL statement.</param>
+    /// <returns>The current instance of <see cref="RefreshOptions"/> for method chaining.</returns>
+    /// <exception cref="System.ArgumentException">Thrown when refreshSql is null or whitespace.</exception>
+    public RefreshOptions WithRefreshSql(string refreshSql)
+    {
+#if NET8_0_OR_GREATER
+        ArgumentException.ThrowIfNullOrWhiteSpace(refreshSql);
+#else
+        ThrowHelper.ThrowIfNullOrWhiteSpace(refreshSql, nameof(refreshSql));
+#endif
+        RefreshSql = refreshSql;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the refresh mode to use for this refresh.
+    /// </summary>
+    /// <param name="refreshMode">The refresh mode.</param>
+    /// <returns>The current instance of <see cref="RefreshOptions"/> for method chaining.</returns>
+    public RefreshOptions WithRefreshMode(RefreshMode refreshMode)
+    {
+        RefreshMode = refreshMode;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the maximum amount of jitter to add before starting this refresh.
+    /// </summary>
+    /// <param name="maxJitter">The maximum jitter. Must not be negative.</param>
+    /// <returns>The current instance of <see cref="RefreshOptions"/> for method chaining.</returns>
+    /// <exception cref="System.ArgumentOutOfRangeException">Thrown when maxJitter is negative.</exception>
+    public RefreshOptions WithMaxJitter(TimeSpan maxJitter)
+    {
+        if (maxJitter < TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(nameof(maxJitter), maxJitter, "maxJitter must not be negative.");
+        }
+
+        MaxJitter = maxJitter;
+        return this;
+    }
+
+    /// <summary>
+    /// Serializes the options to the JSON request body expected by the Spice runtime.
+    /// Unset options are omitted so the runtime falls back to the dataset configuration.
+    /// </summary>
+    internal string ToJson()
+    {
+        var body = new Dictionary<string, string>(StringComparer.Ordinal);
+
+        if (!string.IsNullOrWhiteSpace(RefreshSql))
+        {
+            body["refresh_sql"] = RefreshSql!;
+        }
+
+        if (RefreshMode.HasValue)
+        {
+            body["refresh_mode"] = ToWireValue(RefreshMode.Value);
+        }
+
+        if (MaxJitter.HasValue)
+        {
+            body["refresh_jitter_max"] = FormatDuration(MaxJitter.Value);
+        }
+
+        return JsonSerializer.Serialize(body);
+    }
+
+    /// <summary>
+    /// Maps a <see cref="Datasets.RefreshMode"/> to the value understood by the runtime.
+    /// </summary>
+    internal static string ToWireValue(RefreshMode refreshMode) => refreshMode switch
+    {
+        Datasets.RefreshMode.Full => "full",
+        Datasets.RefreshMode.Append => "append",
+        _ => throw new ArgumentOutOfRangeException(nameof(refreshMode), refreshMode, "Unsupported refresh mode."),
+    };
+
+    /// <summary>
+    /// Formats a <see cref="TimeSpan"/> as a duration string the runtime can parse (for example "10s" or "1500ms").
+    /// </summary>
+    internal static string FormatDuration(TimeSpan value)
+    {
+        var totalMilliseconds = (long)value.TotalMilliseconds;
+
+        return totalMilliseconds % 1000 == 0
+            ? string.Concat((totalMilliseconds / 1000).ToString(CultureInfo.InvariantCulture), "s")
+            : string.Concat(totalMilliseconds.ToString(CultureInfo.InvariantCulture), "ms");
+    }
+}

Spice/src/Http/ISpiceHttpClient.cs

@@ -20,6 +20,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
+using Spice.Datasets;
+
 namespace Spice.Http;
 
 /// <summary>
@@ -44,4 +46,15 @@ public interface ISpiceHttpClient : IDisposable
     /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
     /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
     Task RefreshDatasetAsync(string datasetName);
+
+    /// <summary>
+    /// Refreshes a dataset in the Spice runtime, overriding the dataset's configured
+    /// refresh settings for this refresh only.
+    /// </summary>
+    /// <param name="datasetName">The name of the dataset to refresh</param>
+    /// <param name="options">Overrides for this refresh, or null to use the dataset configuration</param>
+    /// <returns>A task representing the asynchronous operation</returns>
+    /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
+    /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
+    Task RefreshDatasetAsync(string datasetName, RefreshOptions? options);
 }

Spice/src/Http/SpiceHttpClient.cs

@@ -23,6 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 using System.Net.Http.Headers;
 using System.Text;
 using Spice.Auth;
+using Spice.Datasets;
 
 namespace Spice.Http;
 
@@ -121,7 +122,18 @@ public Task<string> QueryAsync(string sql)
     /// <returns>A task representing the asynchronous operation</returns>
     /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
     /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
-    public async Task RefreshDatasetAsync(string datasetName)
+    public Task RefreshDatasetAsync(string datasetName) => RefreshDatasetAsync(datasetName, null);
+
+    /// <summary>
+    /// Refreshes a dataset in the Spice runtime, overriding the dataset's configured
+    /// refresh settings for this refresh only.
+    /// </summary>
+    /// <param name="datasetName">The name of the dataset to refresh</param>
+    /// <param name="options">Overrides for this refresh, or null to use the dataset configuration</param>
+    /// <returns>A task representing the asynchronous operation</returns>
+    /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
+    /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
+    public async Task RefreshDatasetAsync(string datasetName, RefreshOptions? options)
     {
 #if NET8_0_OR_GREATER
         ObjectDisposedException.ThrowIf(_disposed, this);
@@ -132,7 +144,8 @@ public async Task RefreshDatasetAsync(string datasetName)
 #endif
 
         var url = $"{_httpAddress}/v1/datasets/{datasetName}/acceleration/refresh";
-        var response = await _httpClient.PostAsync(url, null).ConfigureAwait(false);
+        using var content = new StringContent(options?.ToJson() ?? "{}", Encoding.UTF8, "application/json");
+        var response = await _httpClient.PostAsync(url, content).ConfigureAwait(false);
         response.EnsureSuccessStatusCode();
     }
 

Spice/src/SpiceClient.cs

@@ -24,6 +24,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 using Apache.Arrow.Ipc;
 using Spice.Adbc;
 using Spice.Config;
+using Spice.Datasets;
 using Spice.Flight;
 using Spice.Http;
 
@@ -171,13 +172,32 @@ public Task<FlightClientRecordBatchStreamReader> Query(string sql)
     }
 
     /// <summary>
-    /// Refreshes a dataset in the Spice runtime.
+    /// Refreshes a dataset in the Spice runtime using the dataset's configured refresh settings.
     /// </summary>
     /// <param name="datasetName">The name of the dataset to refresh</param>
     /// <returns>A task representing the asynchronous operation</returns>
     /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
     /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
-    public Task RefreshDatasetAsync(string datasetName)
+    public Task RefreshDatasetAsync(string datasetName) => RefreshDatasetAsync(datasetName, null);
+
+    /// <summary>
+    /// Refreshes a dataset in the Spice runtime, overriding the dataset's configured
+    /// refresh settings for this refresh only.
+    /// </summary>
+    /// <example>
+    /// <code>
+    /// // Refresh only the recent rows, appending them to the accelerated data.
+    /// await client.RefreshDatasetAsync("taxi_trips", new RefreshOptions()
+    ///     .WithRefreshSql("SELECT * FROM taxi_trips WHERE tip_amount &gt; 10.0")
+    ///     .WithRefreshMode(RefreshMode.Append));
+    /// </code>
+    /// </example>
+    /// <param name="datasetName">The name of the dataset to refresh</param>
+    /// <param name="options">Overrides for this refresh, or null to use the dataset configuration</param>
+    /// <returns>A task representing the asynchronous operation</returns>
+    /// <exception cref="System.ArgumentException">Thrown when datasetName is null or empty</exception>
+    /// <exception cref="System.Net.Http.HttpRequestException">Thrown when the HTTP request fails</exception>
+    public Task RefreshDatasetAsync(string datasetName, RefreshOptions? options)
     {
 #if NET8_0_OR_GREATER
         ObjectDisposedException.ThrowIf(_disposed, this);
@@ -186,7 +206,7 @@ public Task RefreshDatasetAsync(string datasetName)
 #endif
         if (HttpClient == null) throw new InvalidOperationException("HttpClient not initialized");
 
-        return HttpClient.RefreshDatasetAsync(datasetName);
+        return HttpClient.RefreshDatasetAsync(datasetName, options);
     }
 
     private bool _disposed;