From 268bf663f87671183f33e3cdb4a0e17ce1e489f9 Mon Sep 17 00:00:00 2001 From: Michael Harris Date: Thu, 19 Feb 2026 14:50:49 -0700 Subject: [PATCH 1/4] feat(auth): allow providing raw private key and email instead of filepath --- CMakeLists.txt | 1 + docs/pages/index.md | 8 +++ src/gsheets_auth.cpp | 36 ++++++++---- src/gsheets_copy.cpp | 53 +++-------------- src/include/utils/options.hpp | 24 ++++++++ src/sheets/auth/service_account_auth.cpp | 10 +++- src/sheets/auth_factory.cpp | 1 + src/utils/options.cpp | 75 ++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 59 deletions(-) create mode 100644 src/include/utils/options.hpp create mode 100644 src/utils/options.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index da939ef..c3f970c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ set(EXTENSION_SOURCES src/sheets/range.cpp src/sheets/auth_factory.cpp src/utils/secret.cpp + src/utils/options.cpp src/utils/proxy.cpp src/utils/version.cpp) diff --git a/docs/pages/index.md b/docs/pages/index.md index d175837..d7fb367 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -50,6 +50,14 @@ CREATE SECRET ( PROVIDER key_file, FILEPATH '' ); + +-- OR without a file path +CREATE SECRET ( + TYPE gsheet, + PROVIDER key_file, + EMAIL '', + SECRET '' +); ``` ### HTTP Proxy diff --git a/src/gsheets_auth.cpp b/src/gsheets_auth.cpp index b421eb4..142dbe8 100644 --- a/src/gsheets_auth.cpp +++ b/src/gsheets_auth.cpp @@ -2,10 +2,11 @@ #include #include -#include "duckdb.hpp" +#include "duckdb/common/exception/binder_exception.hpp" #include "gsheets_auth.hpp" #include "gsheets_utils.hpp" +#include "utils/options.hpp" using json = nlohmann::json; @@ -30,7 +31,6 @@ static void RedactCommonKeys(KeyValueSecret &result) { result.redact_keys.insert("proxy_password"); } -// TODO: Maybe this should be a KeyValueSecret static unique_ptr CreateGsheetSecretFromAccessToken(ClientContext &context, CreateSecretInput &input) { auto scope = input.scope; @@ -63,23 +63,31 @@ static unique_ptr CreateGsheetSecretFromOAuth(ClientContext &context return std::move(result); } -// TODO: Maybe this should be a KeyValueSecret static unique_ptr CreateGsheetSecretFromKeyFile(ClientContext &context, CreateSecretInput &input) { auto scope = input.scope; auto result = make_uniq(scope, input.type, input.provider, input.name); - // Want to store the private key and email in case the secret is persisted - std::string filepath_key = "filepath"; - auto filepath = (input.options.find(filepath_key)->second).ToString(); - - std::ifstream ifs(filepath); - if (!ifs.is_open()) { - throw IOException("Could not open JSON key file at: " + filepath); + std::string email, secret; + auto filepath = duckdb::sheets::GetStringOption(input.options, "filepath"); + if (filepath.empty()) { + email = duckdb::sheets::GetStringOption(input.options, "email"); + if (email.empty()) { + throw BinderException("Must provide email if not using filepath"); + } + secret = duckdb::sheets::GetStringOption(input.options, "secret"); + if (email.empty()) { + throw BinderException("Must provide secret value if not using filepath"); + } + } else { + std::ifstream ifs(filepath); + if (!ifs.is_open()) { + throw IOException("Could not open JSON key file at: " + filepath); + } + json credentials_file = json::parse(ifs); + email = credentials_file["client_email"].get(); + secret = credentials_file["private_key"].get(); } - json credentials_file = json::parse(ifs); - std::string email = credentials_file["client_email"].get(); - std::string secret = credentials_file["private_key"].get(); // Manage specific secret option (*result).secret_map["email"] = Value(email); @@ -119,6 +127,8 @@ void CreateGsheetSecretFunctions::Register(ExtensionLoader &loader) { // Register the key_file secret provider CreateSecretFunction key_file_function = {type, "key_file", CreateGsheetSecretFromKeyFile, {}}; key_file_function.named_parameters["filepath"] = LogicalType::VARCHAR; + key_file_function.named_parameters["email"] = LogicalType::VARCHAR; + key_file_function.named_parameters["secret"] = LogicalType::VARCHAR; RegisterCommonSecretParameters(key_file_function); loader.RegisterSecretType(secret_type); diff --git a/src/gsheets_copy.cpp b/src/gsheets_copy.cpp index eaab03c..d0041f6 100644 --- a/src/gsheets_copy.cpp +++ b/src/gsheets_copy.cpp @@ -1,6 +1,5 @@ #include -#include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/exception/binder_exception.hpp" #include "duckdb/common/file_system.hpp" @@ -10,6 +9,8 @@ #include "gsheets_copy.hpp" #include "gsheets_utils.hpp" +#include "utils/options.hpp" + #include "sheets/auth_factory.hpp" #include "sheets/client.hpp" #include "sheets/exception.hpp" @@ -26,44 +27,6 @@ GSheetCopyFunction::GSheetCopyFunction() : CopyFunction("gsheet") { copy_to_sink = GSheetWriteSink; } -static std::string GetStringOption(const case_insensitive_map_t> &options, const std::string &name, - const std::string &default_value = "") { - const auto it = options.find(name); - if (it == options.end()) { - return default_value; - } - std::string err; - Value val; - if (!it->second.back().DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { - throw BinderException(name + " option must be VARCHAR"); - } - if (val.IsNull()) { - throw BinderException(name + " option must not be NULL"); - } - return StringValue::Get(val); -} - -// NOTE: the second value in pair is a flag indicating if the value was set by the user -static std::pair GetBoolOption(const case_insensitive_map_t> &options, - const std::string &name, bool default_value = false) { - const auto it = options.find(name); - if (it == options.end()) { - return std::make_pair(default_value, false); - } - if (it->second.size() != 1) { - throw BinderException(name + " option must be a single boolean value"); - } - std::string err; - Value val; - if (!it->second.back().DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { - throw BinderException(name + " option must be a single boolean value"); - } - if (val.IsNull()) { - throw BinderException(name + " option must be a single boolean value"); - } - return std::make_pair(BooleanValue::Get(val), true); -} - unique_ptr GSheetCopyFunction::GSheetWriteBind(ClientContext &context, CopyFunctionBindInput &input, const vector &names, const vector &sql_types) { @@ -71,13 +34,13 @@ unique_ptr GSheetCopyFunction::GSheetWriteBind(ClientContext &cont string file_path = input.info.file_path; auto options = input.info.options; - auto sheet = GetStringOption(options, "sheet"); - auto range = GetStringOption(options, "range"); - bool overwrite_sheet = GetBoolOption(options, "overwrite_sheet", true).first; - bool overwrite_range = GetBoolOption(options, "overwrite_range", false).first; - bool create_if_not_exists = GetBoolOption(options, "create_if_not_exists", false).first; + auto sheet = duckdb::sheets::GetStringOption(options, "sheet"); + auto range = duckdb::sheets::GetStringOption(options, "range"); + bool overwrite_sheet = duckdb::sheets::GetBoolOption(options, "overwrite_sheet", true).first; + bool overwrite_range = duckdb::sheets::GetBoolOption(options, "overwrite_range", false).first; + bool create_if_not_exists = duckdb::sheets::GetBoolOption(options, "create_if_not_exists", false).first; - auto header_result = GetBoolOption(options, "header", true); + auto header_result = duckdb::sheets::GetBoolOption(options, "header", true); bool header = header_result.second ? header_result.first : (overwrite_range || overwrite_sheet); if (create_if_not_exists && sheet.empty()) { diff --git a/src/include/utils/options.hpp b/src/include/utils/options.hpp new file mode 100644 index 0000000..84c34a2 --- /dev/null +++ b/src/include/utils/options.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "duckdb/common/types/value.hpp" + +namespace duckdb { +namespace sheets { + +std::string GetStringOption(const case_insensitive_map_t> &options, const std::string &name, + const std::string &default_value = ""); + +std::string GetStringOption(const case_insensitive_map_t &options, const std::string &name, + const std::string &default_value = ""); + +std::pair GetBoolOption(const case_insensitive_map_t> &options, const std::string &name, + bool default_value = false); + +std::pair GetBoolOption(const case_insensitive_map_t &options, const std::string &name, + bool default_value = false); + +} // namespace sheets +} // namespace duckdb diff --git a/src/sheets/auth/service_account_auth.cpp b/src/sheets/auth/service_account_auth.cpp index 7fbb5ef..b9aff20 100644 --- a/src/sheets/auth/service_account_auth.cpp +++ b/src/sheets/auth/service_account_auth.cpp @@ -66,8 +66,16 @@ std::string ServiceAccountAuth::CreateJwt() { std::string claimsB64 = Base64UrlEncode(claimSet.dump()); std::string signInput = headerB64 + "." + claimsB64; + // Replace literal "\n" with actual newlines + std::string pem = privateKey; + size_t pos = 0; + while ((pos = pem.find("\\n", pos)) != std::string::npos) { + pem.replace(pos, 2, "\n"); + pos += 1; + } + // Parse PEM private key into EVP_PKEY (RAII handles cleanup) - BIOPtr bio(BIO_new_mem_buf(privateKey.c_str(), -1)); + BIOPtr bio(BIO_new_mem_buf(pem.c_str(), -1)); if (!bio) { throw duckdb::IOException("Failed to create BIO for private key"); } diff --git a/src/sheets/auth_factory.cpp b/src/sheets/auth_factory.cpp index 1d97094..3e2b0b0 100644 --- a/src/sheets/auth_factory.cpp +++ b/src/sheets/auth_factory.cpp @@ -3,6 +3,7 @@ #include "utils/secret.hpp" #include "sheets/auth/bearer_token_auth.hpp" #include "sheets/auth/service_account_auth.hpp" +#include namespace duckdb { namespace sheets { diff --git a/src/utils/options.cpp b/src/utils/options.cpp new file mode 100644 index 0000000..b59c57f --- /dev/null +++ b/src/utils/options.cpp @@ -0,0 +1,75 @@ +#include "duckdb/common/exception/binder_exception.hpp" +#include "duckdb/common/types/value.hpp" + +#include "utils/options.hpp" + +std::string duckdb::sheets::GetStringOption(const case_insensitive_map_t> &options, + const std::string &name, const std::string &default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return default_value; + } + std::string err; + Value val; + if (!it->second.back().DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { + throw BinderException(name + " option must be VARCHAR"); + } + if (val.IsNull()) { + throw BinderException(name + " option must not be NULL"); + } + return StringValue::Get(val); +} + +std::string duckdb::sheets::GetStringOption(const case_insensitive_map_t &options, const std::string &name, + const std::string &default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return default_value; + } + std::string err; + Value val; + if (!it->second.DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { + throw BinderException(name + " option must be VARCHAR"); + } + if (val.IsNull()) { + throw BinderException(name + " option must not be NULL"); + } + return StringValue::Get(val); +} + +std::pair duckdb::sheets::GetBoolOption(const case_insensitive_map_t> &options, + const std::string &name, bool default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return std::make_pair(default_value, false); + } + if (it->second.size() != 1) { + throw BinderException(name + " option must be a single boolean value"); + } + std::string err; + Value val; + if (!it->second.back().DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { + throw BinderException(name + " option must be a single boolean value"); + } + if (val.IsNull()) { + throw BinderException(name + " option must be a single boolean value"); + } + return std::make_pair(BooleanValue::Get(val), true); +} + +std::pair duckdb::sheets::GetBoolOption(const case_insensitive_map_t &options, + const std::string &name, bool default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return std::make_pair(default_value, false); + } + std::string err; + Value val; + if (!it->second.DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { + throw BinderException(name + " option must be a single boolean value"); + } + if (val.IsNull()) { + throw BinderException(name + " option must be a single boolean value"); + } + return std::make_pair(BooleanValue::Get(val), true); +} From 1304847b60aedbcec09bae79079e1700db1b0e38 Mon Sep 17 00:00:00 2001 From: Michael Harris Date: Thu, 19 Feb 2026 15:16:27 -0700 Subject: [PATCH 2/4] chore(auth): remove redundant import --- src/sheets/auth_factory.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sheets/auth_factory.cpp b/src/sheets/auth_factory.cpp index 3e2b0b0..1d97094 100644 --- a/src/sheets/auth_factory.cpp +++ b/src/sheets/auth_factory.cpp @@ -3,7 +3,6 @@ #include "utils/secret.hpp" #include "sheets/auth/bearer_token_auth.hpp" #include "sheets/auth/service_account_auth.hpp" -#include namespace duckdb { namespace sheets { From 7883d7b76216835a2304ea1f2a8d54cf9691309c Mon Sep 17 00:00:00 2001 From: Michael Harris Date: Fri, 20 Feb 2026 13:37:04 -0700 Subject: [PATCH 3/4] fix(docs): fix comment --- docs/pages/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/index.md b/docs/pages/index.md index d7fb367..93de5c2 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -51,7 +51,7 @@ CREATE SECRET ( FILEPATH '' ); --- OR without a file path +-- OR by passing the secret directly CREATE SECRET ( TYPE gsheet, PROVIDER key_file, From 52f329e018d0490c01e11fbf6b2b8917b8add939 Mon Sep 17 00:00:00 2001 From: Michael Harris Date: Fri, 20 Feb 2026 13:59:48 -0700 Subject: [PATCH 4/4] chore(encoding): factor out pem key normalization and test --- src/include/sheets/util/encoding.hpp | 2 ++ src/sheets/auth/service_account_auth.cpp | 8 +------- src/sheets/util/encoding.cpp | 10 ++++++++++ test/unit/sheets/util/test_encoding.cpp | 20 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/include/sheets/util/encoding.hpp b/src/include/sheets/util/encoding.hpp index cca879c..26f5885 100644 --- a/src/include/sheets/util/encoding.hpp +++ b/src/include/sheets/util/encoding.hpp @@ -16,5 +16,7 @@ std::string Base64UrlEncode(const unsigned char *data, size_t len); std::string Base64UrlEncode(const std::string &input); +std::string NormalizePemKey(const std::string &key); + } // namespace sheets } // namespace duckdb diff --git a/src/sheets/auth/service_account_auth.cpp b/src/sheets/auth/service_account_auth.cpp index b9aff20..056dc58 100644 --- a/src/sheets/auth/service_account_auth.cpp +++ b/src/sheets/auth/service_account_auth.cpp @@ -66,13 +66,7 @@ std::string ServiceAccountAuth::CreateJwt() { std::string claimsB64 = Base64UrlEncode(claimSet.dump()); std::string signInput = headerB64 + "." + claimsB64; - // Replace literal "\n" with actual newlines - std::string pem = privateKey; - size_t pos = 0; - while ((pos = pem.find("\\n", pos)) != std::string::npos) { - pem.replace(pos, 2, "\n"); - pos += 1; - } + auto pem = NormalizePemKey(privateKey); // Parse PEM private key into EVP_PKEY (RAII handles cleanup) BIOPtr bio(BIO_new_mem_buf(pem.c_str(), -1)); diff --git a/src/sheets/util/encoding.cpp b/src/sheets/util/encoding.cpp index bb9d4d2..f2663f5 100644 --- a/src/sheets/util/encoding.cpp +++ b/src/sheets/util/encoding.cpp @@ -42,5 +42,15 @@ std::string Base64UrlEncode(const std::string &input) { return Base64UrlEncode(reinterpret_cast(input.c_str()), input.length()); } +std::string NormalizePemKey(const std::string &key) { + std::string pem = key; + size_t pos = 0; + while ((pos = pem.find("\\n", pos)) != std::string::npos) { + pem.replace(pos, 2, "\n"); + pos += 1; + } + return pem; +} + } // namespace sheets } // namespace duckdb diff --git a/test/unit/sheets/util/test_encoding.cpp b/test/unit/sheets/util/test_encoding.cpp index 1da0d82..285d14d 100644 --- a/test/unit/sheets/util/test_encoding.cpp +++ b/test/unit/sheets/util/test_encoding.cpp @@ -67,3 +67,23 @@ TEST_CASE("Base64UrlEncode JWT header", "[encoding]") { std::string header = R"({"alg":"RS256","typ":"JWT"})"; REQUIRE(duckdb::sheets::Base64UrlEncode(header) == "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9"); } + +TEST_CASE("NormalizePemKey Key with literal \\n sequences") { + std::string input = "-----BEGIN PRIVATE KEY-----\\nMIIE...\\n-----END PRIVATE KEY-----\\n"; + std::string expected = "-----BEGIN PRIVATE KEY-----\nMIIE...\n-----END PRIVATE KEY-----\n"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == expected); +} + +TEST_CASE("NormalizePemKey Key already has real newlines") { + std::string input = "-----BEGIN PRIVATE KEY-----\nMIIE...\n-----END PRIVATE KEY-----\n"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == input); +} + +TEST_CASE("NormalizePemKey empty string") { + REQUIRE(duckdb::sheets::NormalizePemKey("") == ""); +} + +TEST_CASE("NormalizePemKey no newlines at all") { + std::string input = "just-a-string-with-no-newlines"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == input); +}