diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 5935195..301b0b6 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -3,7 +3,6 @@ # name: Main Extension Distribution Pipeline on: - push: pull_request: workflow_dispatch: diff --git a/.github/workflows/SQLTests.yml b/.github/workflows/SQLTests.yml index ca6e69d..1ae4874 100644 --- a/.github/workflows/SQLTests.yml +++ b/.github/workflows/SQLTests.yml @@ -6,9 +6,6 @@ on: GSHEETS_KEY_FILE_JSON: required: false - # For fork PRs: runs with base repo secrets after environment approval - pull_request_target: - # Manual trigger: maintainers can run SQL tests for any ref workflow_dispatch: inputs: diff --git a/.gitignore b/.gitignore index c4a7041..1c8d250 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,9 @@ _site/ token.txt credentials.json service-account-credentials.json + +# Local Temp Files +tmp/ + +# Proxy Configs +tinyproxy.conf diff --git a/CMakeLists.txt b/CMakeLists.txt index 6581756..da939ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,10 +31,12 @@ set(EXTENSION_SOURCES src/sheets/transport/httplib_client.cpp src/sheets/transport/duckdb_http_client.cpp src/sheets/transport/mock_http_client.cpp + src/sheets/transport/client_factory.cpp src/sheets/util/encoding.cpp src/sheets/range.cpp src/sheets/auth_factory.cpp src/utils/secret.cpp + src/utils/proxy.cpp src/utils/version.cpp) # Warn on unused/dead code (GCC/Clang only; MSVC uses different flag syntax) diff --git a/docs/pages/index.md b/docs/pages/index.md index a1b381b..990b66e 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -9,9 +9,8 @@ hide_title: true **🚧 Experimental 🚧** Here be dragons - - + A DuckDB extension for reading and writing Google Sheets with SQL. @@ -26,7 +25,7 @@ LOAD gsheets; The latest version of [DuckDB](https://duckdb.org/docs/installation) (currently 1.4.0) is supported. -## Usage +## Usage ### Authenticate @@ -37,22 +36,45 @@ CREATE SECRET (TYPE gsheet); -- OR create a secret with your Google API access token (boring, see below guide) CREATE SECRET ( - TYPE gsheet, - PROVIDER access_token, + TYPE gsheet, + PROVIDER access_token, TOKEN '' ); --- OR create a non-expiring JSON secret with your Google API private key +-- OR create a non-expiring JSON secret with your Google API private key -- (This enables use in non-interactive workflows like data pipelines) -- (see "Getting a Google API Access Private Key" below) CREATE SECRET ( - TYPE gsheet, - PROVIDER key_file, + TYPE gsheet, + PROVIDER key_file, FILEPATH '' ); ``` +### HTTP Proxy + +See [HTTP Proxy documentation](https://duckdb.org/docs/stable/core_extensions/httpfs/https#http-proxy). + +You can add an HTTP proxy using the Secrets Manager: + +```sql +CREATE SECRET http_proxy ( + TYPE http, + HTTP_PROXY 'http_proxy_url', + HTTP_PROXY_USERNAME 'username', + HTTP_PROXY_PASSWORD 'password' +); +``` + +Alternatively, you can add it via configuration options: + +```sql +SET http_proxy = 'http_proxy_url'; +SET http_proxy_username = 'username'; +SET http_proxy_password = 'password'; +``` + ### Read ```sql @@ -102,40 +124,40 @@ copy to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry- -- Write a spreadsheet to a specific sheet with the sheet parameter -- NOTE: A sheet parameter will take precedence over the query string -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot'); -- Write a spreadsheet to a specific range with the range parameter -- NOTE: A range parameter will take precedence over the query string -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot', range 'B2:C10000'); -- Only overwrite the range that is being written to -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_range TRUE); -- Overwrite the entire sheet (this is the default) -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet TRUE); -- Append below existing data -- (by default, no header will be included) -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet FALSE, overwrite_range FALSE); -- Append below existing data, but force the output of a header -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, sheet 'Woot', range 'B2:C10000', overwrite_sheet FALSE, overwrite_range FALSE, header TRUE); -- Write with no header -copy -to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' +copy +to 'https://docs.google.com/spreadsheets/d/11QdEasMWbETbFVxry-SsD8jVcdYIT1zBQszcF84MdE8/edit' (format gsheet, header FALSE); ``` @@ -178,7 +200,7 @@ The contents of the JSON file will be stored in the secret, as will the temporar Follow steps 13 and 14. -This private key by default will not expire. Use caution with it. +This private key by default will not expire. Use caution with it. This will also require an additional API request approximately every 30 minutes. @@ -188,6 +210,6 @@ This will also require an additional API request approximately every 30 minutes. - Google Sheets has a limit of 10,000,000 cells per spreadsheet. - Sheets must already exist to COPY TO them. -## Support +## Support If you are having problems, find a bug, or have an idea for an improvement, please [file an issue on GitHub](https://github.com/evidence-dev/duckdb_gsheets). diff --git a/src/gsheets_copy.cpp b/src/gsheets_copy.cpp index 33f6b52..7b3472f 100644 --- a/src/gsheets_copy.cpp +++ b/src/gsheets_copy.cpp @@ -10,7 +10,7 @@ #include "sheets/auth_factory.hpp" #include "sheets/client.hpp" #include "sheets/range.hpp" -#include "sheets/transport/httplib_client.hpp" +#include "sheets/transport/client_factory.hpp" #include "sheets/types.hpp" namespace duckdb { @@ -139,7 +139,7 @@ unique_ptr GSheetCopyFunction::GSheetWriteInitializeGlobal(C std::string sheet_range; // Initialize client - auto http = make_uniq(); + auto http = sheets::CreateHttpClient(context); auto auth = sheets::CreateAuthFromSecret(context, *http); if (!auth) { throw InvalidInputException("No 'gsheet' secret found..."); diff --git a/src/gsheets_read.cpp b/src/gsheets_read.cpp index 52d8c57..1d946af 100644 --- a/src/gsheets_read.cpp +++ b/src/gsheets_read.cpp @@ -7,7 +7,7 @@ #include "sheets/client.hpp" #include "sheets/auth_factory.hpp" -#include "sheets/transport/httplib_client.hpp" +#include "sheets/transport/client_factory.hpp" namespace duckdb { @@ -103,12 +103,12 @@ unique_ptr ReadSheetBind(ClientContext &context, TableFunctionBind std::string sheet_range = extract_sheet_range(sheet_input); // Initialize client - sheets::HttpLibClient http; - auto auth = sheets::CreateAuthFromSecret(context, http); + auto http = sheets::CreateHttpClient(context); + auto auth = sheets::CreateAuthFromSecret(context, *http); if (!auth) { throw InvalidInputException("No 'gsheet' secret found..."); } - sheets::GoogleSheetsClient client(http, *auth); + sheets::GoogleSheetsClient client(*http, *auth); // Parse named parameters for (auto &kv : input.named_parameters) { diff --git a/src/include/sheets/auth_factory.hpp b/src/include/sheets/auth_factory.hpp index 622bfa9..5b1c0ab 100644 --- a/src/include/sheets/auth_factory.hpp +++ b/src/include/sheets/auth_factory.hpp @@ -8,8 +8,7 @@ namespace duckdb { namespace sheets { -std::unique_ptr CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http, - const std::string &secretName = ""); +std::unique_ptr CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http); } // namespace sheets } // namespace duckdb diff --git a/src/include/sheets/transport/client_factory.hpp b/src/include/sheets/transport/client_factory.hpp new file mode 100644 index 0000000..6e06718 --- /dev/null +++ b/src/include/sheets/transport/client_factory.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "duckdb/main/client_context.hpp" + +#include "sheets/transport/http_client.hpp" + +namespace duckdb { +namespace sheets { + +std::unique_ptr CreateHttpClient(ClientContext &ctx); + +} // namespace sheets +} // namespace duckdb diff --git a/src/include/sheets/transport/http_type.hpp b/src/include/sheets/transport/http_type.hpp index 09f2f21..bbf9452 100644 --- a/src/include/sheets/transport/http_type.hpp +++ b/src/include/sheets/transport/http_type.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -22,5 +23,13 @@ struct HttpResponse { HttpHeaders headers; std::string body; }; + +struct HttpProxyConfig { + std::string host; + uint16_t port = 0; + std::string username; + std::string password; +}; + } // namespace sheets } // namespace duckdb diff --git a/src/include/sheets/transport/httplib_client.hpp b/src/include/sheets/transport/httplib_client.hpp index 5330bc5..69f6312 100644 --- a/src/include/sheets/transport/httplib_client.hpp +++ b/src/include/sheets/transport/httplib_client.hpp @@ -1,16 +1,24 @@ #pragma once +#include + #include "sheets/transport/http_client.hpp" +#include "sheets/transport/http_type.hpp" namespace duckdb { namespace sheets { class HttpLibClient : public IHttpClient { public: + explicit HttpLibClient(HttpProxyConfig proxy_config) : proxy_config(std::move(proxy_config)) { + } + HttpResponse Execute(const HttpRequest &request) override; private: void ParseUrl(const std::string &url, std::string &baseUrl, std::string &path); + + HttpProxyConfig proxy_config; }; } // namespace sheets } // namespace duckdb diff --git a/src/include/utils/proxy.hpp b/src/include/utils/proxy.hpp new file mode 100644 index 0000000..ebf5508 --- /dev/null +++ b/src/include/utils/proxy.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include "duckdb/main/client_context.hpp" + +#include "sheets/transport/http_type.hpp" + +namespace duckdb { +namespace sheets { + +HttpProxyConfig GetHttpProxyConfig(ClientContext &ctx); + +} // namespace sheets +} // namespace duckdb diff --git a/src/include/utils/secret.hpp b/src/include/utils/secret.hpp index bcccb10..d8c7a1b 100644 --- a/src/include/utils/secret.hpp +++ b/src/include/utils/secret.hpp @@ -2,11 +2,13 @@ #include "duckdb/main/client_context.hpp" #include "duckdb/main/secret/secret.hpp" +#include "duckdb/main/secret/secret_manager.hpp" namespace duckdb { namespace sheets { -const KeyValueSecret *GetGSheetSecret(ClientContext &ctx, const std::string &secretName = ""); +const SecretMatch GetSecretMatch(ClientContext &ctx, const std::string &path, const std::string &type); +const KeyValueSecret *GetGSheetSecret(ClientContext &ctx); } // namespace sheets } // namespace duckdb diff --git a/src/sheets/auth_factory.cpp b/src/sheets/auth_factory.cpp index e6451ba..1d97094 100644 --- a/src/sheets/auth_factory.cpp +++ b/src/sheets/auth_factory.cpp @@ -7,9 +7,8 @@ namespace duckdb { namespace sheets { -std::unique_ptr CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http, - const std::string &secretName) { - auto *secret = GetGSheetSecret(ctx, secretName); +std::unique_ptr CreateAuthFromSecret(ClientContext &ctx, IHttpClient &http) { + auto *secret = GetGSheetSecret(ctx); if (!secret) { return nullptr; } diff --git a/src/sheets/transport/client_factory.cpp b/src/sheets/transport/client_factory.cpp new file mode 100644 index 0000000..9b73abe --- /dev/null +++ b/src/sheets/transport/client_factory.cpp @@ -0,0 +1,17 @@ +#include "duckdb/common/helper.hpp" +#include "duckdb/main/client_context.hpp" + +#include "sheets/transport/http_client.hpp" +#include "sheets/transport/httplib_client.hpp" +#include "utils/proxy.hpp" + +namespace duckdb { +namespace sheets { + +std::unique_ptr CreateHttpClient(ClientContext &ctx) { + auto proxy_config = GetHttpProxyConfig(ctx); + return make_uniq(proxy_config); +} + +} // namespace sheets +} // namespace duckdb diff --git a/src/sheets/transport/httplib_client.cpp b/src/sheets/transport/httplib_client.cpp index 06a82eb..20abdff 100644 --- a/src/sheets/transport/httplib_client.cpp +++ b/src/sheets/transport/httplib_client.cpp @@ -32,6 +32,12 @@ HttpResponse HttpLibClient::Execute(const HttpRequest &request) { ParseUrl(request.url, baseUrl, path); duckdb_httplib_openssl::Client client(baseUrl); + if (!proxy_config.host.empty()) { + client.set_proxy(proxy_config.host, proxy_config.port); + if (!proxy_config.username.empty()) { + client.set_proxy_basic_auth(proxy_config.username, proxy_config.password); + } + } // Extract content-type from request headers (default to application/json) std::string contentType = "application/json"; diff --git a/src/utils/proxy.cpp b/src/utils/proxy.cpp new file mode 100644 index 0000000..deb818a --- /dev/null +++ b/src/utils/proxy.cpp @@ -0,0 +1,98 @@ +#include +#include +#include +#include + +#include "duckdb/common/exception.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/main/secret/secret.hpp" + +#include "utils/proxy.hpp" +#include "utils/secret.hpp" +#include "sheets/transport/http_type.hpp" + +namespace duckdb { +namespace sheets { + +static void ParseHTTPProxyHost(string &proxy_value, string &hostname_out, uint16_t &port_out) { + uint16_t default_port = 80; + auto sanitized_proxy_value = proxy_value; + if (StringUtil::StartsWith(proxy_value, "http://")) { + sanitized_proxy_value = proxy_value.substr(7); + } else if (StringUtil::StartsWith(proxy_value, "https://")) { + default_port = 443; + sanitized_proxy_value = proxy_value.substr(8); + } + + // Remove all trailing slashes to avoid issues with host path + while (!sanitized_proxy_value.empty() && StringUtil::EndsWith(sanitized_proxy_value, "/")) { + sanitized_proxy_value.pop_back(); + } + + auto proxy_split = StringUtil::Split(sanitized_proxy_value, ":"); + if (proxy_split.size() == 1) { + hostname_out = proxy_split[0]; + port_out = default_port; + } else if (proxy_split.size() == 2) { + uint16_t port; + try { + auto val = std::stoul(proxy_split[1]); + if (val > std::numeric_limits::max()) { + throw InvalidInputException("Failed to parse port from http_proxy '%s'", proxy_value); + } + port = static_cast(val); + } catch (const std::invalid_argument &e) { + throw InvalidInputException("Failed to parse port from http_proxy '%s'", proxy_value); + } catch (const std::out_of_range &e) { + throw InvalidInputException("Failed to parse port from http_proxy '%s'", proxy_value); + } + hostname_out = proxy_split[0]; + port_out = port; + } else { + throw InvalidInputException("Failed to parse http_proxy '%s' into a host and port", proxy_value); + } +} + +HttpProxyConfig GetHttpProxyConfig(ClientContext &ctx) { + HttpProxyConfig proxy_config; + auto match = GetSecretMatch(ctx, "", "http"); + if (match.HasMatch()) { + auto &secret = match.GetSecret(); + auto http_secret = dynamic_cast(&secret); + + Value http_proxy, http_proxy_username, http_proxy_password; + + if (http_secret->TryGetValue("http_proxy", http_proxy)) { + auto proxy_value = http_proxy.ToString(); + if (!proxy_value.empty()) { + ParseHTTPProxyHost(proxy_value, proxy_config.host, proxy_config.port); + + if (http_secret->TryGetValue("http_proxy_username", http_proxy_username)) { + proxy_config.username = http_proxy_username.ToString(); + } + + if (http_secret->TryGetValue("http_proxy_password", http_proxy_password)) { + proxy_config.password = http_proxy_password.ToString(); + } + } + } + } else { + // try falling back on configs + Value http_proxy, http_proxy_username, http_proxy_password; + ctx.TryGetCurrentSetting("http_proxy", http_proxy); + + auto proxy_value = http_proxy.ToString(); + + if (!proxy_value.empty()) { + ParseHTTPProxyHost(proxy_value, proxy_config.host, proxy_config.port); + ctx.TryGetCurrentSetting("http_proxy_username", http_proxy_username); + ctx.TryGetCurrentSetting("http_proxy_password", http_proxy_password); + proxy_config.username = http_proxy_username.ToString(); + proxy_config.password = http_proxy_password.ToString(); + } + } + return proxy_config; +} + +} // namespace sheets +} // namespace duckdb diff --git a/src/utils/secret.cpp b/src/utils/secret.cpp index 7179d0a..7f7fd57 100644 --- a/src/utils/secret.cpp +++ b/src/utils/secret.cpp @@ -1,15 +1,20 @@ #include "utils/secret.hpp" +#include "duckdb/main/secret/secret.hpp" #include "duckdb/main/secret/secret_manager.hpp" #include "duckdb/catalog/catalog_transaction.hpp" namespace duckdb { namespace sheets { -const KeyValueSecret *GetGSheetSecret(ClientContext &ctx, const std::string &secretName) { +const SecretMatch GetSecretMatch(ClientContext &ctx, const std::string &path, const std::string &type) { auto &manager = SecretManager::Get(ctx); auto transaction = CatalogTransaction::GetSystemCatalogTransaction(ctx); - auto match = manager.LookupSecret(transaction, "gsheet", "gsheet"); + return manager.LookupSecret(transaction, path, type); +} + +const KeyValueSecret *GetGSheetSecret(ClientContext &ctx) { + auto match = GetSecretMatch(ctx, "gsheet", "gsheet"); if (!match.HasMatch()) { return nullptr; }