diff --git a/CMakeLists.txt b/CMakeLists.txt index da939ef..c3f970c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ set(EXTENSION_SOURCES src/sheets/range.cpp src/sheets/auth_factory.cpp src/utils/secret.cpp + src/utils/options.cpp src/utils/proxy.cpp src/utils/version.cpp) diff --git a/docs/pages/index.md b/docs/pages/index.md index d175837..93de5c2 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -50,6 +50,14 @@ CREATE SECRET ( PROVIDER key_file, FILEPATH '' ); + +-- OR by passing the secret directly +CREATE SECRET ( + TYPE gsheet, + PROVIDER key_file, + EMAIL '', + SECRET '' +); ``` ### HTTP Proxy diff --git a/src/gsheets_auth.cpp b/src/gsheets_auth.cpp index b421eb4..142dbe8 100644 --- a/src/gsheets_auth.cpp +++ b/src/gsheets_auth.cpp @@ -2,10 +2,11 @@ #include #include -#include "duckdb.hpp" +#include "duckdb/common/exception/binder_exception.hpp" #include "gsheets_auth.hpp" #include "gsheets_utils.hpp" +#include "utils/options.hpp" using json = nlohmann::json; @@ -30,7 +31,6 @@ static void RedactCommonKeys(KeyValueSecret &result) { result.redact_keys.insert("proxy_password"); } -// TODO: Maybe this should be a KeyValueSecret static unique_ptr CreateGsheetSecretFromAccessToken(ClientContext &context, CreateSecretInput &input) { auto scope = input.scope; @@ -63,23 +63,31 @@ static unique_ptr CreateGsheetSecretFromOAuth(ClientContext &context return std::move(result); } -// TODO: Maybe this should be a KeyValueSecret static unique_ptr CreateGsheetSecretFromKeyFile(ClientContext &context, CreateSecretInput &input) { auto scope = input.scope; auto result = make_uniq(scope, input.type, input.provider, input.name); - // Want to store the private key and email in case the secret is persisted - std::string filepath_key = "filepath"; - auto filepath = (input.options.find(filepath_key)->second).ToString(); - - std::ifstream ifs(filepath); - if (!ifs.is_open()) { - throw IOException("Could not open JSON key file at: " + filepath); + std::string email, secret; + auto filepath = duckdb::sheets::GetStringOption(input.options, "filepath"); + if (filepath.empty()) { + email = duckdb::sheets::GetStringOption(input.options, "email"); + if (email.empty()) { + throw BinderException("Must provide email if not using filepath"); + } + secret = duckdb::sheets::GetStringOption(input.options, "secret"); + if (email.empty()) { + throw BinderException("Must provide secret value if not using filepath"); + } + } else { + std::ifstream ifs(filepath); + if (!ifs.is_open()) { + throw IOException("Could not open JSON key file at: " + filepath); + } + json credentials_file = json::parse(ifs); + email = credentials_file["client_email"].get(); + secret = credentials_file["private_key"].get(); } - json credentials_file = json::parse(ifs); - std::string email = credentials_file["client_email"].get(); - std::string secret = credentials_file["private_key"].get(); // Manage specific secret option (*result).secret_map["email"] = Value(email); @@ -119,6 +127,8 @@ void CreateGsheetSecretFunctions::Register(ExtensionLoader &loader) { // Register the key_file secret provider CreateSecretFunction key_file_function = {type, "key_file", CreateGsheetSecretFromKeyFile, {}}; key_file_function.named_parameters["filepath"] = LogicalType::VARCHAR; + key_file_function.named_parameters["email"] = LogicalType::VARCHAR; + key_file_function.named_parameters["secret"] = LogicalType::VARCHAR; RegisterCommonSecretParameters(key_file_function); loader.RegisterSecretType(secret_type); diff --git a/src/gsheets_copy.cpp b/src/gsheets_copy.cpp index eaab03c..d0041f6 100644 --- a/src/gsheets_copy.cpp +++ b/src/gsheets_copy.cpp @@ -1,6 +1,5 @@ #include -#include "duckdb/common/case_insensitive_map.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/common/exception/binder_exception.hpp" #include "duckdb/common/file_system.hpp" @@ -10,6 +9,8 @@ #include "gsheets_copy.hpp" #include "gsheets_utils.hpp" +#include "utils/options.hpp" + #include "sheets/auth_factory.hpp" #include "sheets/client.hpp" #include "sheets/exception.hpp" @@ -26,44 +27,6 @@ GSheetCopyFunction::GSheetCopyFunction() : CopyFunction("gsheet") { copy_to_sink = GSheetWriteSink; } -static std::string GetStringOption(const case_insensitive_map_t> &options, const std::string &name, - const std::string &default_value = "") { - const auto it = options.find(name); - if (it == options.end()) { - return default_value; - } - std::string err; - Value val; - if (!it->second.back().DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { - throw BinderException(name + " option must be VARCHAR"); - } - if (val.IsNull()) { - throw BinderException(name + " option must not be NULL"); - } - return StringValue::Get(val); -} - -// NOTE: the second value in pair is a flag indicating if the value was set by the user -static std::pair GetBoolOption(const case_insensitive_map_t> &options, - const std::string &name, bool default_value = false) { - const auto it = options.find(name); - if (it == options.end()) { - return std::make_pair(default_value, false); - } - if (it->second.size() != 1) { - throw BinderException(name + " option must be a single boolean value"); - } - std::string err; - Value val; - if (!it->second.back().DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { - throw BinderException(name + " option must be a single boolean value"); - } - if (val.IsNull()) { - throw BinderException(name + " option must be a single boolean value"); - } - return std::make_pair(BooleanValue::Get(val), true); -} - unique_ptr GSheetCopyFunction::GSheetWriteBind(ClientContext &context, CopyFunctionBindInput &input, const vector &names, const vector &sql_types) { @@ -71,13 +34,13 @@ unique_ptr GSheetCopyFunction::GSheetWriteBind(ClientContext &cont string file_path = input.info.file_path; auto options = input.info.options; - auto sheet = GetStringOption(options, "sheet"); - auto range = GetStringOption(options, "range"); - bool overwrite_sheet = GetBoolOption(options, "overwrite_sheet", true).first; - bool overwrite_range = GetBoolOption(options, "overwrite_range", false).first; - bool create_if_not_exists = GetBoolOption(options, "create_if_not_exists", false).first; + auto sheet = duckdb::sheets::GetStringOption(options, "sheet"); + auto range = duckdb::sheets::GetStringOption(options, "range"); + bool overwrite_sheet = duckdb::sheets::GetBoolOption(options, "overwrite_sheet", true).first; + bool overwrite_range = duckdb::sheets::GetBoolOption(options, "overwrite_range", false).first; + bool create_if_not_exists = duckdb::sheets::GetBoolOption(options, "create_if_not_exists", false).first; - auto header_result = GetBoolOption(options, "header", true); + auto header_result = duckdb::sheets::GetBoolOption(options, "header", true); bool header = header_result.second ? header_result.first : (overwrite_range || overwrite_sheet); if (create_if_not_exists && sheet.empty()) { diff --git a/src/include/sheets/util/encoding.hpp b/src/include/sheets/util/encoding.hpp index cca879c..26f5885 100644 --- a/src/include/sheets/util/encoding.hpp +++ b/src/include/sheets/util/encoding.hpp @@ -16,5 +16,7 @@ std::string Base64UrlEncode(const unsigned char *data, size_t len); std::string Base64UrlEncode(const std::string &input); +std::string NormalizePemKey(const std::string &key); + } // namespace sheets } // namespace duckdb diff --git a/src/include/utils/options.hpp b/src/include/utils/options.hpp new file mode 100644 index 0000000..84c34a2 --- /dev/null +++ b/src/include/utils/options.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "duckdb/common/types/value.hpp" + +namespace duckdb { +namespace sheets { + +std::string GetStringOption(const case_insensitive_map_t> &options, const std::string &name, + const std::string &default_value = ""); + +std::string GetStringOption(const case_insensitive_map_t &options, const std::string &name, + const std::string &default_value = ""); + +std::pair GetBoolOption(const case_insensitive_map_t> &options, const std::string &name, + bool default_value = false); + +std::pair GetBoolOption(const case_insensitive_map_t &options, const std::string &name, + bool default_value = false); + +} // namespace sheets +} // namespace duckdb diff --git a/src/sheets/auth/service_account_auth.cpp b/src/sheets/auth/service_account_auth.cpp index 7fbb5ef..056dc58 100644 --- a/src/sheets/auth/service_account_auth.cpp +++ b/src/sheets/auth/service_account_auth.cpp @@ -66,8 +66,10 @@ std::string ServiceAccountAuth::CreateJwt() { std::string claimsB64 = Base64UrlEncode(claimSet.dump()); std::string signInput = headerB64 + "." + claimsB64; + auto pem = NormalizePemKey(privateKey); + // Parse PEM private key into EVP_PKEY (RAII handles cleanup) - BIOPtr bio(BIO_new_mem_buf(privateKey.c_str(), -1)); + BIOPtr bio(BIO_new_mem_buf(pem.c_str(), -1)); if (!bio) { throw duckdb::IOException("Failed to create BIO for private key"); } diff --git a/src/sheets/util/encoding.cpp b/src/sheets/util/encoding.cpp index bb9d4d2..f2663f5 100644 --- a/src/sheets/util/encoding.cpp +++ b/src/sheets/util/encoding.cpp @@ -42,5 +42,15 @@ std::string Base64UrlEncode(const std::string &input) { return Base64UrlEncode(reinterpret_cast(input.c_str()), input.length()); } +std::string NormalizePemKey(const std::string &key) { + std::string pem = key; + size_t pos = 0; + while ((pos = pem.find("\\n", pos)) != std::string::npos) { + pem.replace(pos, 2, "\n"); + pos += 1; + } + return pem; +} + } // namespace sheets } // namespace duckdb diff --git a/src/utils/options.cpp b/src/utils/options.cpp new file mode 100644 index 0000000..b59c57f --- /dev/null +++ b/src/utils/options.cpp @@ -0,0 +1,75 @@ +#include "duckdb/common/exception/binder_exception.hpp" +#include "duckdb/common/types/value.hpp" + +#include "utils/options.hpp" + +std::string duckdb::sheets::GetStringOption(const case_insensitive_map_t> &options, + const std::string &name, const std::string &default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return default_value; + } + std::string err; + Value val; + if (!it->second.back().DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { + throw BinderException(name + " option must be VARCHAR"); + } + if (val.IsNull()) { + throw BinderException(name + " option must not be NULL"); + } + return StringValue::Get(val); +} + +std::string duckdb::sheets::GetStringOption(const case_insensitive_map_t &options, const std::string &name, + const std::string &default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return default_value; + } + std::string err; + Value val; + if (!it->second.DefaultTryCastAs(LogicalType::VARCHAR, val, &err)) { + throw BinderException(name + " option must be VARCHAR"); + } + if (val.IsNull()) { + throw BinderException(name + " option must not be NULL"); + } + return StringValue::Get(val); +} + +std::pair duckdb::sheets::GetBoolOption(const case_insensitive_map_t> &options, + const std::string &name, bool default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return std::make_pair(default_value, false); + } + if (it->second.size() != 1) { + throw BinderException(name + " option must be a single boolean value"); + } + std::string err; + Value val; + if (!it->second.back().DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { + throw BinderException(name + " option must be a single boolean value"); + } + if (val.IsNull()) { + throw BinderException(name + " option must be a single boolean value"); + } + return std::make_pair(BooleanValue::Get(val), true); +} + +std::pair duckdb::sheets::GetBoolOption(const case_insensitive_map_t &options, + const std::string &name, bool default_value) { + const auto it = options.find(name); + if (it == options.end()) { + return std::make_pair(default_value, false); + } + std::string err; + Value val; + if (!it->second.DefaultTryCastAs(LogicalType::BOOLEAN, val, &err)) { + throw BinderException(name + " option must be a single boolean value"); + } + if (val.IsNull()) { + throw BinderException(name + " option must be a single boolean value"); + } + return std::make_pair(BooleanValue::Get(val), true); +} diff --git a/test/unit/sheets/util/test_encoding.cpp b/test/unit/sheets/util/test_encoding.cpp index 1da0d82..285d14d 100644 --- a/test/unit/sheets/util/test_encoding.cpp +++ b/test/unit/sheets/util/test_encoding.cpp @@ -67,3 +67,23 @@ TEST_CASE("Base64UrlEncode JWT header", "[encoding]") { std::string header = R"({"alg":"RS256","typ":"JWT"})"; REQUIRE(duckdb::sheets::Base64UrlEncode(header) == "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9"); } + +TEST_CASE("NormalizePemKey Key with literal \\n sequences") { + std::string input = "-----BEGIN PRIVATE KEY-----\\nMIIE...\\n-----END PRIVATE KEY-----\\n"; + std::string expected = "-----BEGIN PRIVATE KEY-----\nMIIE...\n-----END PRIVATE KEY-----\n"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == expected); +} + +TEST_CASE("NormalizePemKey Key already has real newlines") { + std::string input = "-----BEGIN PRIVATE KEY-----\nMIIE...\n-----END PRIVATE KEY-----\n"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == input); +} + +TEST_CASE("NormalizePemKey empty string") { + REQUIRE(duckdb::sheets::NormalizePemKey("") == ""); +} + +TEST_CASE("NormalizePemKey no newlines at all") { + std::string input = "just-a-string-with-no-newlines"; + REQUIRE(duckdb::sheets::NormalizePemKey(input) == input); +}