diff --git a/CMakeLists.txt b/CMakeLists.txt index d36aa5f..08065ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,10 @@ set(EXTENSION_SOURCES build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) +if (EMSCRIPTEN) + set (DUCKDB_EXTENSION_GSHEETS_LINKED_LIBS "../../vcpkg_installed/wasm32-emscripten/lib/lib*.a") +endif() + # Link OpenSSL in both the static library as the loadable extension target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto) @@ -35,3 +39,4 @@ install( EXPORT "${DUCKDB_EXPORT_SET}" LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") + diff --git a/src/gsheets_copy.cpp b/src/gsheets_copy.cpp index 50b1f21..c1e2997 100644 --- a/src/gsheets_copy.cpp +++ b/src/gsheets_copy.cpp @@ -177,7 +177,7 @@ unique_ptr GSheetCopyFunction::GSheetWriteInitializeGlobal(C if (!options.sheet.empty()) { sheet_name = options.sheet; } else { - sheet_name = get_sheet_name_from_id(spreadsheet_id, sheet_id, token); + sheet_name = get_sheet_name_from_id(context, spreadsheet_id, sheet_id, token); } if (!options.range.empty()) { @@ -191,10 +191,10 @@ unique_ptr GSheetCopyFunction::GSheetWriteInitializeGlobal(C // Do this here in the initialization so that it only happens once // OVERWRITE_RANGE takes precedence since it defaults to false and is less destructive if (options.overwrite_range) { - delete_sheet_data(spreadsheet_id, token, encoded_sheet_name, sheet_range); + delete_sheet_data(context, spreadsheet_id, token, encoded_sheet_name, sheet_range); } else if (options.overwrite_sheet) { std::string empty_string = ""; // An empty string will clear the entire sheet - delete_sheet_data(spreadsheet_id, token, encoded_sheet_name, empty_string); + delete_sheet_data(context, spreadsheet_id, token, encoded_sheet_name, empty_string); } // Write out the headers to the file here in the Initialize so they are only written once @@ -217,7 +217,7 @@ unique_ptr GSheetCopyFunction::GSheetWriteInitializeGlobal(C // If we are appending, header defaults to false if (options.header) { std::string response = - call_sheets_api(spreadsheet_id, token, encoded_sheet_name, sheet_range, HttpMethod::POST, request_body); + call_sheets_api(context, spreadsheet_id, token, encoded_sheet_name, sheet_range, HttpMethod::POST, request_body); // Check for errors in the response json response_json = parseJson(response); @@ -249,7 +249,7 @@ void GSheetCopyFunction::GSheetWriteSink(ExecutionContext &context, FunctionData if (!options.sheet.empty()) { sheet_name = options.sheet; } else { - sheet_name = get_sheet_name_from_id(gstate.spreadsheet_id, sheet_id, gstate.token); + sheet_name = get_sheet_name_from_id(context.client, gstate.spreadsheet_id, sheet_id, gstate.token); } if (!options.range.empty()) { @@ -286,7 +286,7 @@ void GSheetCopyFunction::GSheetWriteSink(ExecutionContext &context, FunctionData std::string request_body = sheet_data.dump(); // Make the API call to write data to the Google Sheet - std::string response = call_sheets_api(gstate.spreadsheet_id, gstate.token, encoded_sheet_name, sheet_range, + std::string response = call_sheets_api(context.client, gstate.spreadsheet_id, gstate.token, encoded_sheet_name, sheet_range, HttpMethod::POST, request_body); // Check for errors in the response diff --git a/src/gsheets_extension.cpp b/src/gsheets_extension.cpp index 0e15312..1b9d73b 100644 --- a/src/gsheets_extension.cpp +++ b/src/gsheets_extension.cpp @@ -48,10 +48,12 @@ unique_ptr ReadSheetReplacement(ClientContext &context, ReplacementSca } static void LoadInternal(ExtensionLoader &loader) { +#ifndef EMSCRIPTEN // Initialize OpenSSL SSL_library_init(); SSL_load_error_strings(); OpenSSL_add_all_algorithms(); +#endif // Register read_gsheet table function TableFunction read_gsheet_function("read_gsheet", {LogicalType::VARCHAR}, ReadSheetFunction, ReadSheetBind); @@ -67,6 +69,8 @@ static void LoadInternal(ExtensionLoader &loader) { CreateGsheetSecretFunctions::Register(loader); auto &config = DBConfig::GetConfig(loader.GetDatabaseInstance()); + config.AddExtensionOption("gsheets_endpoint", "Alternative endpoint for the sheets.googleapis.com API", + LogicalType::VARCHAR, Value("sheets.googleapis.com")); config.replacement_scans.emplace_back(ReadSheetReplacement); } diff --git a/src/gsheets_get_token.cpp b/src/gsheets_get_token.cpp index 4cd9198..c020830 100644 --- a/src/gsheets_get_token.cpp +++ b/src/gsheets_get_token.cpp @@ -137,7 +137,7 @@ TokenDetails get_token(ClientContext &context, const KeyValueSecret *kv_secret) std::string body = "grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=" + std::string(jwt); - std::string response = perform_https_request("oauth2.googleapis.com", "/token", "", HttpMethod::POST, + std::string response = perform_https_request(context, "oauth2.googleapis.com", "/token", "", HttpMethod::POST, body, "application/x-www-form-urlencoded"); json response_json = parseJson(response); std::string token = response_json["access_token"].get(); diff --git a/src/gsheets_read.cpp b/src/gsheets_read.cpp index 17ec684..913bbff 100644 --- a/src/gsheets_read.cpp +++ b/src/gsheets_read.cpp @@ -14,10 +14,10 @@ namespace duckdb { using json = nlohmann::json; ReadSheetBindData::ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name, - string sheet_range) + string sheet_range, ClientContext &context) : spreadsheet_id(spreadsheet_id), token(token), finished(false), row_index(0), header(header), sheet_name(sheet_name), sheet_range(sheet_range) { - response = call_sheets_api(spreadsheet_id, token, sheet_name, sheet_range, HttpMethod::GET); + response = call_sheets_api(context, spreadsheet_id, token, sheet_name, sheet_range, HttpMethod::GET); } bool IsValidNumber(const string &value) { @@ -195,7 +195,7 @@ unique_ptr ReadSheetBind(ClientContext &context, TableFunctionBind } // Validate that sheet with name exists for better error messaging - sheet_id = get_sheet_id_from_name(spreadsheet_id, sheet_name, token); + sheet_id = get_sheet_id_from_name(context, spreadsheet_id, sheet_name, token); } else if (kv.first == "range") { sheet_range = kv.second.GetValue(); } @@ -206,15 +206,15 @@ unique_ptr ReadSheetBind(ClientContext &context, TableFunctionBind sheet_id = extract_sheet_id(sheet_input); if (sheet_id.empty()) { // Fallback to first sheet by index - sheet_name = get_sheet_name_from_index(spreadsheet_id, "0", token); + sheet_name = get_sheet_name_from_index(context, spreadsheet_id, "0", token); } else { - sheet_name = get_sheet_name_from_id(spreadsheet_id, sheet_id, token); + sheet_name = get_sheet_name_from_id(context, spreadsheet_id, sheet_id, token); } } std::string encoded_sheet_name = url_encode(sheet_name); - auto bind_data = make_uniq(spreadsheet_id, token, header, encoded_sheet_name, sheet_range); + auto bind_data = make_uniq(spreadsheet_id, token, header, encoded_sheet_name, sheet_range, context); json cleanJson = parseJson(bind_data->response); SheetData sheet_data = getSheetData(cleanJson); diff --git a/src/gsheets_requests.cpp b/src/gsheets_requests.cpp index 28c4ab9..557dea7 100644 --- a/src/gsheets_requests.cpp +++ b/src/gsheets_requests.cpp @@ -1,5 +1,8 @@ #include "gsheets_requests.hpp" #include "duckdb/common/exception.hpp" +#include "duckdb/common/http_util.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" #include #include #include @@ -8,100 +11,60 @@ namespace duckdb { -std::string perform_https_request(const std::string &host, const std::string &path, const std::string &token, - HttpMethod method, const std::string &body, const std::string &content_type) { - // NOTE: implements an exponential backoff retry strategy as per https://developers.google.com/sheets/api/limits - const int MAX_RETRIES = 10; - int retry_count = 0; - int backoff_s = 1; - while (retry_count < MAX_RETRIES) { - SSL_CTX *ctx = SSL_CTX_new(TLS_client_method()); - if (!ctx) { - throw duckdb::IOException("Failed to create SSL context"); - } - - BIO *bio = BIO_new_ssl_connect(ctx); - SSL *ssl; - BIO_get_ssl(bio, &ssl); - SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY); - - BIO_set_conn_hostname(bio, (host + ":443").c_str()); - - if (BIO_do_connect(bio) <= 0) { - BIO_free_all(bio); - SSL_CTX_free(ctx); - throw duckdb::IOException("Failed to connect"); - } - - std::string method_str; - switch (method) { - case HttpMethod::GET: - method_str = "GET"; - break; - case HttpMethod::POST: - method_str = "POST"; - break; - case HttpMethod::PUT: - method_str = "PUT"; - break; - } - - std::string request = method_str + " " + path + " HTTP/1.0\r\n"; - request += "Host: " + host + "\r\n"; - request += "Authorization: Bearer " + token + "\r\n"; - request += "Connection: close\r\n"; - - if (!body.empty()) { - request += "Content-Type: " + content_type + "\r\n"; - request += "Content-Length: " + std::to_string(body.length()) + "\r\n"; - } - - request += "\r\n"; - - if (!body.empty()) { - request += body; - } - - if (BIO_write(bio, request.c_str(), request.length()) <= 0) { - BIO_free_all(bio); - SSL_CTX_free(ctx); - throw duckdb::IOException("Failed to write request"); - } - - std::string response; - char buffer[1024]; - int len; - while ((len = BIO_read(bio, buffer, sizeof(buffer))) > 0) { - response.append(buffer, len); - } - - BIO_free_all(bio); - SSL_CTX_free(ctx); - - // Check for rate limit exceeded - if (response.find("429 Too Many Requests") != std::string::npos) { - std::this_thread::sleep_for(std::chrono::seconds(backoff_s)); - backoff_s *= 2; - retry_count++; - continue; - } - - // Extract body from response - size_t body_start = response.find("\r\n\r\n"); - if (body_start != std::string::npos) { - return response.substr(body_start + 4); - } - - return response; - } - // TODO: refactor HTTPS to catch more than just rate limit exceeded or else - // this could give false positives. - throw duckdb::IOException("Google Sheets API rate limit exceeded"); + +std::string perform_https_request(ClientContext &context, const std::string &host, const std::string &path, + const std::string &token, HttpMethod method, const std::string &body, + const std::string &content_type) { + auto &db = DatabaseInstance::GetDatabase(context); + + HTTPHeaders headers(db); + headers.Insert("Host", StringUtil::Format("%s", "http://127.0.0.1:8080/")); + headers.Insert("Authorization", StringUtil::Format("Bearer %s", token)); + // headers.Insert("Connection", StringUtil::Format("close")); + + if (!body.empty()) { + headers.Insert("Content-Type", StringUtil::Format("%s", content_type)); + headers.Insert("Content-Length", StringUtil::Format("%s", std::to_string(body.length()))); + } + auto &http_util = HTTPUtil::Get(db); + unique_ptr params; + std::string x = string("https://") + host + path; + params = http_util.InitializeParameters(context, x); + // Unclear what's peculiar about extension install flow, but those two parameters are needed + // to avoid lengthy retry on 304 + params->follow_location = false; + params->keep_alive = false; + + switch (method) { + case HttpMethod::GET: { + + GetRequestInfo get_request(x, headers, *params, nullptr, nullptr); + get_request.try_request = true; + + auto response = http_util.Request(get_request); + + return response->body; + + break; + } + case HttpMethod::POST: + case HttpMethod::PUT: + throw IOException("POST or PUT not implemented"); + break; + } + + // TODO: refactor HTTPS to catch more than just rate limit exceeded or else + // this could give false positives. + throw duckdb::IOException("Google Sheets API rate limit exceeded"); } -std::string call_sheets_api(const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, +std::string call_sheets_api(ClientContext &context, const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, const std::string &sheet_range, HttpMethod method, const std::string &body) { std::string host = "sheets.googleapis.com"; + Value value; + if (context.TryGetCurrentSetting("gsheets_endpoint", value)) { + host = value.ToString(); + } std::string path = "/v4/spreadsheets/" + spreadsheet_id + "/values/" + sheet_name; if (!sheet_range.empty()) { @@ -113,21 +76,29 @@ std::string call_sheets_api(const std::string &spreadsheet_id, const std::string path += "?valueInputOption=USER_ENTERED"; } - return perform_https_request(host, path, token, method, body); + return perform_https_request(context, host, path, token, method, body); } -std::string delete_sheet_data(const std::string &spreadsheet_id, const std::string &token, +std::string delete_sheet_data(ClientContext &context, const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, const std::string &sheet_range) { std::string host = "sheets.googleapis.com"; + Value value; + if (context.TryGetCurrentSetting("gsheets_endpoint", value)) { + host = value.ToString(); + } std::string sheet_and_range = sheet_range.empty() ? sheet_name : sheet_name + "!" + sheet_range; std::string path = "/v4/spreadsheets/" + spreadsheet_id + "/values/" + sheet_and_range + ":clear"; - return perform_https_request(host, path, token, HttpMethod::POST, "{}"); + return perform_https_request(context, host, path, token, HttpMethod::POST, "{}"); } -std::string get_spreadsheet_metadata(const std::string &spreadsheet_id, const std::string &token) { +std::string get_spreadsheet_metadata(ClientContext &context, const std::string &spreadsheet_id, const std::string &token) { std::string host = "sheets.googleapis.com"; + Value value; + if (context.TryGetCurrentSetting("gsheets_endpoint", value)) { + host = value.ToString(); + } std::string path = "/v4/spreadsheets/" + spreadsheet_id + "?&fields=sheets.properties"; - return perform_https_request(host, path, token, HttpMethod::GET, ""); + return perform_https_request(context, host, path, token, HttpMethod::GET, ""); } } // namespace duckdb diff --git a/src/gsheets_utils.cpp b/src/gsheets_utils.cpp index b457515..c3d75df 100644 --- a/src/gsheets_utils.cpp +++ b/src/gsheets_utils.cpp @@ -51,9 +51,9 @@ std::string extract_sheet_range(const std::string &input) { return ""; } -std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std::string &sheet_id, +std::string get_sheet_name_from_id(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_id, const std::string &token) { - std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token); + std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token); json metadata = parseJson(metadata_response); for (const auto &sheet : metadata["sheets"]) { if (sheet["properties"]["sheetId"].get() == std::stoi(sheet_id)) { @@ -63,9 +63,9 @@ std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std: throw duckdb::InvalidInputException("Sheet with ID %s not found", sheet_id); } -std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const std::string &sheet_index, +std::string get_sheet_name_from_index(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_index, const std::string &token) { - std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token); + std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token); json metadata = parseJson(metadata_response); for (const auto &sheet : metadata["sheets"]) { if (sheet["properties"]["index"].get() == std::stoi(sheet_index)) { @@ -75,9 +75,9 @@ std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const s throw duckdb::InvalidInputException("Sheet with index %s not found", sheet_index); } -std::string get_sheet_id_from_name(const std::string &spreadsheet_id, const std::string &sheet_name, +std::string get_sheet_id_from_name(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_name, const std::string &token) { - std::string metadata_response = get_spreadsheet_metadata(spreadsheet_id, token); + std::string metadata_response = get_spreadsheet_metadata(context, spreadsheet_id, token); json metadata = parseJson(metadata_response); for (const auto &sheet : metadata["sheets"]) { if (sheet["properties"]["title"].get() == sheet_name) { diff --git a/src/include/gsheets_read.hpp b/src/include/gsheets_read.hpp index cae2a87..ecac480 100644 --- a/src/include/gsheets_read.hpp +++ b/src/include/gsheets_read.hpp @@ -19,7 +19,7 @@ struct ReadSheetBindData : public TableFunctionData { vector return_types; vector names; - ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name, string sheet_range); + ReadSheetBindData(string spreadsheet_id, string token, bool header, string sheet_name, string sheet_range, ClientContext &context); }; void ReadSheetFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output); diff --git a/src/include/gsheets_requests.hpp b/src/include/gsheets_requests.hpp index 6f08857..ab65379 100644 --- a/src/include/gsheets_requests.hpp +++ b/src/include/gsheets_requests.hpp @@ -4,18 +4,20 @@ namespace duckdb { +class ClientContext; + enum class HttpMethod { GET, POST, PUT }; -std::string perform_https_request(const std::string &host, const std::string &path, const std::string &token, +std::string perform_https_request(ClientContext &context, const std::string &host, const std::string &path, const std::string &token, HttpMethod method = HttpMethod::GET, const std::string &body = "", const std::string &content_type = "application/json"); -std::string call_sheets_api(const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, +std::string call_sheets_api(ClientContext &context, const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, const std::string &sheet_range, HttpMethod method = HttpMethod::GET, const std::string &body = ""); -std::string delete_sheet_data(const std::string &spreadsheet_id, const std::string &token, +std::string delete_sheet_data(ClientContext &context, const std::string &spreadsheet_id, const std::string &token, const std::string &sheet_name, const std::string &sheet_range); -std::string get_spreadsheet_metadata(const std::string &spreadsheet_id, const std::string &token); +std::string get_spreadsheet_metadata(ClientContext &context, const std::string &spreadsheet_id, const std::string &token); } // namespace duckdb diff --git a/src/include/gsheets_utils.hpp b/src/include/gsheets_utils.hpp index 14c95ac..e5e0ef8 100644 --- a/src/include/gsheets_utils.hpp +++ b/src/include/gsheets_utils.hpp @@ -9,6 +9,8 @@ using json = nlohmann::json; namespace duckdb { +class ClientContext; + /** * Extracts the sheet ID from a Google Sheets URL or returns the input if it's already a sheet ID. * @param input A Google Sheets URL or sheet ID @@ -38,7 +40,7 @@ std::string extract_sheet_range(const std::string &input); * @param token The Google API token * @return The sheet name */ -std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std::string &sheet_id, +std::string get_sheet_name_from_id(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_id, const std::string &token); /** @@ -48,7 +50,7 @@ std::string get_sheet_name_from_id(const std::string &spreadsheet_id, const std: * @param token The Google API token * @return The sheet name */ -std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const std::string &sheet_index, +std::string get_sheet_name_from_index(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_index, const std::string &token); /** @@ -58,7 +60,7 @@ std::string get_sheet_name_from_index(const std::string &spreadsheet_id, const s * @param token The Google API token * @return The sheet ID */ -std::string get_sheet_id_from_name(const std::string &spreadsheet_id, const std::string &sheet_name, +std::string get_sheet_id_from_name(ClientContext &context, const std::string &spreadsheet_id, const std::string &sheet_name, const std::string &token); struct SheetData {