feat(transport): support http proxy

.github/workflows/MainDistributionPipeline.yml

@@ -3,7 +3,6 @@
 #
 name: Main Extension Distribution Pipeline
 on:
-  push:
   pull_request:
   workflow_dispatch:
 

.github/workflows/SQLTests.yml

@@ -6,9 +6,6 @@ on:
       GSHEETS_KEY_FILE_JSON:
         required: false
 
-  # For fork PRs: runs with base repo secrets after environment approval
-  pull_request_target:
-
   # Manual trigger: maintainers can run SQL tests for any ref
   workflow_dispatch:
     inputs:

.gitignore

@@ -31,3 +31,9 @@ _site/
 token.txt
 credentials.json
 service-account-credentials.json
+
+# Local Temp Files
+tmp/
+
+# Proxy Configs
+tinyproxy.conf

CMakeLists.txt

@@ -31,10 +31,12 @@ set(EXTENSION_SOURCES
     src/sheets/transport/httplib_client.cpp
     src/sheets/transport/duckdb_http_client.cpp
     src/sheets/transport/mock_http_client.cpp
+    src/sheets/transport/client_factory.cpp
     src/sheets/util/encoding.cpp
     src/sheets/range.cpp
     src/sheets/auth_factory.cpp
     src/utils/secret.cpp
+    src/utils/proxy.cpp
     src/utils/version.cpp)
 
 # Warn on unused/dead code (GCC/Clang only; MSVC uses different flag syntax)

docs/pages/index.md

@@ -9,9 +9,8 @@ hide_title: true
 <Alert status="warning">
 
 **🚧 Experimental 🚧** Here be dragons
-
-</Alert>
 
+</Alert>
 
 A DuckDB extension for reading and writing Google Sheets with SQL.
 
@@ -26,7 +25,7 @@ LOAD gsheets;
 
 The latest version of [DuckDB](https://duckdb.org/docs/installation) (currently 1.4.0) is supported.
 
-## Usage 
+## Usage
 
 ### Authenticate
 
@@ -37,22 +36,45 @@ CREATE SECRET (TYPE gsheet);
 
 -- OR create a secret with your Google API access token (boring, see below guide)
 CREATE SECRET (
-    TYPE gsheet, 
-    PROVIDER access_token, 
+    TYPE gsheet,
+    PROVIDER access_token,
     TOKEN '<your_token>'
 );
 
 
--- OR create a non-expiring JSON secret with your Google API private key 
+-- OR create a non-expiring JSON secret with your Google API private key
 -- (This enables use in non-interactive workflows like data pipelines)
 -- (see "Getting a Google API Access Private Key" below)
 CREATE SECRET (
-    TYPE gsheet, 
-    PROVIDER key_file, 
+    TYPE gsheet,
+    PROVIDER key_file,
     FILEPATH '<path_to_JSON_file_with_private_key>'
 );
 ```
 
+### HTTP Proxy
+
+See [HTTP Proxy documentation](https://duckdb.org/docs/stable/core_extensions/httpfs/https#http-proxy).
+
+You can add an HTTP proxy using the Secrets Manager:
+
+```sql
+CREATE SECRET http_proxy (
+    TYPE http,
+    HTTP_PROXY 'http_proxy_url',
+    HTTP_PROXY_USERNAME 'username',
+    HTTP_PROXY_PASSWORD 'password'
+);
+```
+
+Alternatively, you can add it via configuration options:
+
+```sql
+SET http_proxy = 'http_proxy_url';
+SET http_proxy_username = 'username';
+SET http_proxy_password = 'password';
+```
+
 ### Read
 
 ```sql
@@ -102,40 +124,40 @@ copy <table_name> to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-
 
 -- Write a spreadsheet to a specific sheet with the sheet parameter
 -- NOTE: A sheet parameter will take precedence over the query string
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot');
 
 -- Write a spreadsheet to a specific range with the range parameter
 -- NOTE: A range parameter will take precedence over the query string
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot', range 'B2:C10000');
 
 -- Only overwrite the range that is being written to
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_range TRUE);
 
 -- Overwrite the entire sheet (this is the default)
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet TRUE);
 
 -- Append below existing data
 -- (by default, no header will be included)
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet FALSE, overwrite_range FALSE);
 
 -- Append below existing data, but force the output of a header
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet FALSE, overwrite_range FALSE, header TRUE);
 
 -- Write with no header
-copy <table_name> 
-to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' 
+copy <table_name>
+to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit'
 (format gsheet, header FALSE);
 ```
 
@@ -178,7 +200,7 @@ The contents of the JSON file will be stored in the secret, as will the temporar
 
 Follow steps 13 and 14.
 
-This private key by default will not expire. Use caution with it. 
+This private key by default will not expire. Use caution with it.
 
 This will also require an additional API request approximately every 30 minutes.
 
@@ -188,6 +210,6 @@ This will also require an additional API request approximately every 30 minutes.
 - Google Sheets has a limit of 10,000,000 cells per spreadsheet.
 - Sheets must already exist to COPY TO them.
 
-## Support 
+## Support
 
 If you are having problems, find a bug, or have an idea for an improvement, please [file an issue on GitHub](https://github.com/evidence-dev/duckdb_gsheets).

src/gsheets_copy.cpp

@@ -10,7 +10,7 @@
 #include "sheets/auth_factory.hpp"
 #include "sheets/client.hpp"
 #include "sheets/range.hpp"
-#include "sheets/transport/httplib_client.hpp"
+#include "sheets/transport/client_factory.hpp"
 #include "sheets/types.hpp"
 
 namespace duckdb {
@@ -139,7 +139,7 @@ unique_ptr<GlobalFunctionData> GSheetCopyFunction::GSheetWriteInitializeGlobal(C
 	std::string sheet_range;
 
 	// Initialize client
-	auto http = make_uniq<sheets::HttpLibClient>();
+	auto http = sheets::CreateHttpClient(context);
 	auto auth = sheets::CreateAuthFromSecret(context, *http);
 	if (!auth) {
 		throw InvalidInputException("No 'gsheet' secret found...");

src/gsheets_read.cpp

@@ -7,7 +7,7 @@
 
 #include "sheets/client.hpp"
 #include "sheets/auth_factory.hpp"
-#include "sheets/transport/httplib_client.hpp"
+#include "sheets/transport/client_factory.hpp"
 
 namespace duckdb {
 
@@ -20,7 +20,7 @@
 	try {
 		// Try to parse as double
 		size_t processed;
 		std::stod(value, &processed);
 		// Ensure the entire string was processed
 		return processed == value.length();
 	} catch (...) {
@@ -103,12 +103,12 @@
 	std::string sheet_range = extract_sheet_range(sheet_input);
 
 	// Initialize client
-	sheets::HttpLibClient http;
-	auto auth = sheets::CreateAuthFromSecret(context, http);
+	auto http = sheets::CreateHttpClient(context);
+	auto auth = sheets::CreateAuthFromSecret(context, *http);
 	if (!auth) {
 		throw InvalidInputException("No 'gsheet' secret found...");
 	}
-	sheets::GoogleSheetsClient client(http, *auth);
+	sheets::GoogleSheetsClient client(*http, *auth);
 
 	// Parse named parameters
 	for (auto &kv : input.named_parameters) {

src/include/sheets/auth_factory.hpp

@@ -8,8 +8,7 @@
 namespace duckdb {
 namespace sheets {
 
-std::unique_ptr<IAuthProvider> CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http,
-                                                    const std::string &secretName = "");
+std::unique_ptr<IAuthProvider> CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http);
 
 } // namespace sheets
 } // namespace duckdb

src/include/sheets/transport/client_factory.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <memory>
+
+#include "duckdb/main/client_context.hpp"
+
+#include "sheets/transport/http_client.hpp"
+
+namespace duckdb {
+namespace sheets {
+
+std::unique_ptr<IHttpClient> CreateHttpClient(ClientContext &ctx);
+
+} // namespace sheets
+} // namespace duckdb

src/include/sheets/transport/http_type.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <string>
 #include <map>
 
@@ -22,5 +23,13 @@ struct HttpResponse {
 	HttpHeaders headers;
 	std::string body;
 };
+
+struct HttpProxyConfig {
+	std::string host;
+	uint16_t port = 0;
+	std::string username;
+	std::string password;
+};
+
 } // namespace sheets
 } // namespace duckdb

src/include/sheets/transport/httplib_client.hpp

@@ -1,16 +1,24 @@
 #pragma once
 
+#include <utility>
+
 #include "sheets/transport/http_client.hpp"
+#include "sheets/transport/http_type.hpp"
 
 namespace duckdb {
 namespace sheets {
 
 class HttpLibClient : public IHttpClient {
 public:
+	explicit HttpLibClient(HttpProxyConfig proxy_config) : proxy_config(std::move(proxy_config)) {
+	}
+
 	HttpResponse Execute(const HttpRequest &request) override;
 
 private:
 	void ParseUrl(const std::string &url, std::string &baseUrl, std::string &path);
+
+	HttpProxyConfig proxy_config;
 };
 } // namespace sheets
 } // namespace duckdb

src/include/utils/proxy.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "duckdb/main/client_context.hpp"
+
+#include "sheets/transport/http_type.hpp"
+
+namespace duckdb {
+namespace sheets {
+
+HttpProxyConfig GetHttpProxyConfig(ClientContext &ctx);
+
+} // namespace sheets
+} // namespace duckdb

src/include/utils/secret.hpp

@@ -2,11 +2,13 @@
 
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/secret/secret.hpp"
+#include "duckdb/main/secret/secret_manager.hpp"
 
 namespace duckdb {
 namespace sheets {
 
-const KeyValueSecret *GetGSheetSecret(ClientContext &ctx, const std::string &secretName = "");
+const SecretMatch GetSecretMatch(ClientContext &ctx, const std::string &path, const std::string &type);
+const KeyValueSecret *GetGSheetSecret(ClientContext &ctx);
 
 } // namespace sheets
 } // namespace duckdb

src/sheets/auth_factory.cpp

@@ -7,9 +7,8 @@
 namespace duckdb {
 namespace sheets {
 
-std::unique_ptr<IAuthProvider> CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http,
-                                                    const std::string &secretName) {
-	auto *secret = GetGSheetSecret(ctx, secretName);
+std::unique_ptr<IAuthProvider> CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http) {
+	auto *secret = GetGSheetSecret(ctx);
 	if (!secret) {
 		return nullptr;
 	}

src/sheets/transport/client_factory.cpp

@@ -0,0 +1,17 @@
+#include "duckdb/common/helper.hpp"
+#include "duckdb/main/client_context.hpp"
+
+#include "sheets/transport/http_client.hpp"
+#include "sheets/transport/httplib_client.hpp"
+#include "utils/proxy.hpp"
+
+namespace duckdb {
+namespace sheets {
+
+std::unique_ptr<IHttpClient> CreateHttpClient(ClientContext &ctx) {
+	auto proxy_config = GetHttpProxyConfig(ctx);
+	return make_uniq<HttpLibClient>(proxy_config);
+}
+
+} // namespace sheets
+} // namespace duckdb

src/sheets/transport/httplib_client.cpp

@@ -32,6 +32,12 @@ HttpResponse HttpLibClient::Execute(const HttpRequest &request) {
 	ParseUrl(request.url, baseUrl, path);
 
 	duckdb_httplib_openssl::Client client(baseUrl);
+	if (!proxy_config.host.empty()) {
+		client.set_proxy(proxy_config.host, proxy_config.port);
+		if (!proxy_config.username.empty()) {
+			client.set_proxy_basic_auth(proxy_config.username, proxy_config.password);
+		}
+	}
 
 	// Extract content-type from request headers (default to application/json)
 	std::string contentType = "application/json";