From 23a9f206bda063a1bebbbd7274e8d48f0ab65039 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 3 Jun 2026 17:32:18 +0200 Subject: [PATCH 1/2] Add ProfileEvents for Iceberg catalogs --- src/Common/ProfileEvents.cpp | 37 +++++++++++++ src/Databases/DataLake/GlueCatalog.cpp | 70 ++++++++++++++++++++++--- src/Databases/DataLake/RestCatalog.cpp | 66 +++++++++++++++++------ src/Databases/DataLake/UnityCatalog.cpp | 63 +++++++++++++++++++--- 4 files changed, 207 insertions(+), 29 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 61a9bdf90e77..277f313074d1 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1418,6 +1418,43 @@ The server successfully detected this situation and will download merged part fr M(ObjectStorageListObjectsCacheMisses, "Number of times object storage list objects operation miss the cache.", ValueType::Number) \ M(ObjectStorageListObjectsCacheExactMatchHits, "Number of times object storage list objects operation hit the cache with an exact match.", ValueType::Number) \ M(ObjectStorageListObjectsCachePrefixMatchHits, "Number of times object storage list objects operation miss the cache using prefix matching.", ValueType::Number) \ + \ + M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetNamespaces, "Number of 'get namespaces' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetNamespacesMicroseconds, "Total time of 'get namespaces' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetTables, "Number of 'get tables' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetTableMetadata, "Number of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + \ + M(DataLakeGlueCatalogGetDatabases, "Number of 'get databases' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetDatabasesMicroseconds, "Total time of 'get databases' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogGetTables, "Number of 'get tables' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogGetTable, "Number of 'get table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetTableMicroseconds, "Total time of 'get table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogCreateDatabase, "Number of 'create database' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogCreateDatabaseMicroseconds, "Total time of 'create database' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogCreateTable, "Number of 'create table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogCreateTableMicroseconds, "Total time of 'create table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogUpdateTable, "Number of 'update table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogDropTable, "Number of 'drop table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + \ + M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetTable, "Number of 'get table' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTableMicroseconds, "Total time of 'get table' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetTableMetadata, "Number of 'get table metadata' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetSchemas, "Number of 'get schemas' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetSchemasMicroseconds, "Total time of 'get schemas' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ #ifdef APPLY_FOR_EXTERNAL_EVENTS diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 047184a1ede3..50b3acd61eee 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -77,6 +78,24 @@ namespace DB::ServerSetting extern const ServerSettingsUInt64 s3_retry_attempts; } +namespace ProfileEvents +{ + extern const Event DataLakeGlueCatalogGetDatabases; + extern const Event DataLakeGlueCatalogGetDatabasesMicroseconds; + extern const Event DataLakeGlueCatalogGetTables; + extern const Event DataLakeGlueCatalogGetTablesMicroseconds; + extern const Event DataLakeGlueCatalogGetTable; + extern const Event DataLakeGlueCatalogGetTableMicroseconds; + extern const Event DataLakeGlueCatalogCreateDatabase; + extern const Event DataLakeGlueCatalogCreateDatabaseMicroseconds; + extern const Event DataLakeGlueCatalogCreateTable; + extern const Event DataLakeGlueCatalogCreateTableMicroseconds; + extern const Event DataLakeGlueCatalogUpdateTable; + extern const Event DataLakeGlueCatalogUpdateTableMicroseconds; + extern const Event DataLakeGlueCatalogDropTable; + extern const Event DataLakeGlueCatalogDropTableMicroseconds; +} + namespace CurrentMetrics { extern const Metric MarkCacheBytes; @@ -184,7 +203,14 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre do { request.SetNextToken(next_token); - auto outcome = glue_client->GetDatabases(request); + + Aws::Glue::Model::GetDatabasesOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetDatabases); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetDatabasesMicroseconds); + outcome = glue_client->GetDatabases(request); + } + if (outcome.IsSuccess()) { const auto & databases_result = outcome.GetResult(); @@ -229,7 +255,12 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t do { request.SetNextToken(next_token); - auto outcome = glue_client->GetTables(request); + Aws::Glue::Model::GetTablesOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTablesMicroseconds); + outcome = glue_client->GetTables(request); + } if (outcome.IsSuccess()) { const auto & tables_result = outcome.GetResult(); @@ -282,6 +313,8 @@ bool GlueCatalog::existsTable(const std::string & database_name, const std::stri request.SetDatabaseName(database_name); request.SetName(table_name); + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); auto outcome = glue_client->GetTable(request); return outcome.IsSuccess(); } @@ -299,7 +332,12 @@ bool GlueCatalog::tryGetTableMetadata( request.SetDatabaseName(database_name); request.SetName(table_name); - auto outcome = glue_client->GetTable(request); + Aws::Glue::Model::GetTableOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); + outcome = glue_client->GetTable(request); + } if (outcome.IsSuccess()) { const auto & table_outcome = outcome.GetResult().GetTable(); @@ -575,6 +613,8 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name) cons db_input.SetName(namespace_name); create_request.SetDatabaseInput(db_input); + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); glue_client->CreateDatabase(create_request); } @@ -612,7 +652,13 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl request.SetTableInput(table_input); - auto response = glue_client->CreateTable(request); + Aws::Glue::Model::CreateTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateTableMicroseconds); + response = glue_client->CreateTable(request); + } if (!response.IsSuccess()) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not create metadata in glue catalog: {}", response.GetError().GetMessage()); @@ -647,7 +693,13 @@ bool GlueCatalog::updateMetadata(const String & namespace_name, const String & t request.SetTableInput(table_input); - auto response = glue_client->UpdateTable(request); + Aws::Glue::Model::UpdateTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUpdateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogUpdateTableMicroseconds); + response = glue_client->UpdateTable(request); + } if (!response.IsSuccess()) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not update metadata in glue catalog {}", response.GetError().GetMessage()); @@ -666,7 +718,13 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ request.SetDatabaseName(namespace_name); request.SetName(table_name); - auto response = glue_client->DeleteTable(request); + Aws::Glue::Model::DeleteTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogDropTableMicroseconds); + response = glue_client->DeleteTable(request); + } if (!response.IsSuccess()) throw DB::Exception( diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 2f3383e5a42a..c80e25df641f 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,20 @@ namespace DB::ErrorCodes extern const int CATALOG_NAMESPACE_DISABLED; } +namespace ProfileEvents +{ + extern const Event DataLakeRestCatalogLoadConfig; + extern const Event DataLakeRestCatalogLoadConfigMicroseconds; + extern const Event DataLakeRestCatalogGetNamespaces; + extern const Event DataLakeRestCatalogGetNamespacesMicroseconds; + extern const Event DataLakeRestCatalogGetTables; + extern const Event DataLakeRestCatalogGetTablesMicroseconds; + extern const Event DataLakeRestCatalogGetTableMetadata; + extern const Event DataLakeRestCatalogGetTableMetadataMicroseconds; + extern const Event DataLakeRestCatalogGetCredentials; + extern const Event DataLakeRestCatalogGetCredentialsMicroseconds; +} + namespace DataLake { @@ -185,10 +200,15 @@ RestCatalog::RestCatalog( RestCatalog::Config RestCatalog::loadConfig() { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; - auto buf = createReadBuffer(CONFIG_ENDPOINT, params); std::string json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); + auto buf = createReadBuffer(CONFIG_ENDPOINT, params); + readJSONObjectPossiblyInvalid(json_str, *buf); + } LOG_DEBUG(log, "Received catalog configuration settings: {}", json_str); @@ -709,6 +729,8 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); auto buf = createReadBuffer(config.prefix / NAMESPACES_ENDPOINT, params); auto namespaces = parseNamespaces(*buf, base_namespace); LOG_DEBUG(log, "Loaded {} namespaces in base namespace {}", namespaces.size(), base_namespace); @@ -799,6 +821,8 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); auto buf = createReadBuffer(config.prefix / endpoint); return parseTables(*buf, base_namespace, limit); } @@ -908,16 +932,21 @@ bool RestCatalog::getTableMetadataImpl( } const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + String json_str; - if (buf->eof()) { - LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); - return false; - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); + auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); - String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + if (buf->eof()) + { + LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); + return false; + } + + readJSONObjectPossiblyInvalid(json_str, *buf); + } #ifdef DEBUG_OR_SANITIZER_BUILD /// This log message might contain credentials, @@ -1255,16 +1284,21 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + String json_str; - if (buf->eof()) { - LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); - return nullptr; - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); + auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); - String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + if (buf->eof()) + { + LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); + return nullptr; + } + + readJSONObjectPossiblyInvalid(json_str, *buf); + } Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(json_str); diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 05709fd14c89..38b627d8638b 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,20 @@ #include #include +namespace ProfileEvents +{ + extern const Event DataLakeUnityCatalogGetTables; + extern const Event DataLakeUnityCatalogGetTablesMicroseconds; + extern const Event DataLakeUnityCatalogGetTable; + extern const Event DataLakeUnityCatalogGetTableMicroseconds; + extern const Event DataLakeUnityCatalogGetTableMetadata; + extern const Event DataLakeUnityCatalogGetTableMetadataMicroseconds; + extern const Event DataLakeUnityCatalogGetCredentials; + extern const Event DataLakeUnityCatalogGetCredentialsMicroseconds; + extern const Event DataLakeUnityCatalogGetSchemas; + extern const Event DataLakeUnityCatalogGetSchemasMicroseconds; +} + namespace DB::ErrorCodes { extern const int DATALAKE_DATABASE_ERROR; @@ -116,7 +131,13 @@ void UnityCatalog::getCredentials(const std::string & table_id, TableMetadata & obj.stringify(os); }; - auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + Poco::Dynamic::Var json; + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetCredentialsMicroseconds); + std::string _; + std::tie(json, _) = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("aws_temp_credentials", object)) @@ -141,7 +162,13 @@ void UnityCatalog::getCredentials(const std::string & table_id, TableMetadata & obj.stringify(os); }; - auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + Poco::Dynamic::Var json; + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetCredentialsMicroseconds); + std::string _; + std::tie(json, _) = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("azure_user_delegation_sas", object)) @@ -173,7 +200,11 @@ bool UnityCatalog::tryGetTableMetadata( std::string json_str; try { - std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / full_table_name); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTableMetadata); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTableMetadataMicroseconds); + std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / full_table_name); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("name", object) && object->get("name").extract() == table_name) { @@ -297,7 +328,11 @@ bool UnityCatalog::existsTable(const std::string & schema_name, const std::strin Poco::Dynamic::Var json; try { - std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / (warehouse + "." + schema_name + "." + table_name)); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTableMicroseconds); + std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / (warehouse + "." + schema_name + "." + table_name)); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("name", object) && object->get("name").extract() == table_name) return true; @@ -325,7 +360,11 @@ DB::Names UnityCatalog::getTablesForSchema(const std::string & schema, size_t li try { - std::tie(json, json_str) = getJSONRequest(TABLES_ENDPOINT, params); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTablesMicroseconds); + std::tie(json, json_str) = getJSONRequest(TABLES_ENDPOINT, params); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (!hasValueAndItsNotNone("tables", object)) @@ -389,7 +428,11 @@ DataLake::ICatalog::Namespaces UnityCatalog::getSchemas(const std::string & base try { - std::tie(json, json_str) = getJSONRequest(SCHEMAS_ENDPOINT, params); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetSchemas); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetSchemasMicroseconds); + std::tie(json, json_str) = getJSONRequest(SCHEMAS_ENDPOINT, params); + } const Poco::JSON::Object::Ptr & object = json.extract(); auto schemas_object = object->get("schemas").extract(); @@ -466,7 +509,13 @@ ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCa { LOG_DEBUG(log, "Update credentials in the catalog"); - auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, {}); + Poco::Dynamic::Var json; + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetCredentialsMicroseconds); + std::string _; + std::tie(json, _) = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, {}); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("aws_temp_credentials", object)) From a2339bf021d5781e389141829e8a3f4264d06673 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 4 Jun 2026 11:41:50 +0200 Subject: [PATCH 2/2] Add more REST events --- src/Common/ProfileEvents.cpp | 8 ++++++++ src/Databases/DataLake/RestCatalog.cpp | 16 ++++++++++++++++ src/Databases/DataLake/S3TablesCatalog.cpp | 9 +++++++++ 3 files changed, 33 insertions(+) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 277f313074d1..993e9d6a85cd 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1429,6 +1429,14 @@ The server successfully detected this situation and will download merged part fr M(DataLakeRestCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Microseconds) \ M(DataLakeRestCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogCreateNamespace, "Number of 'create namespace' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogCreateNamespaceMicroseconds, "Total time of 'create namespace' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogCreateTable, "Number of 'create table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogCreateTableMicroseconds, "Total time of 'create table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogUpdateTable, "Number of 'update table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogDropTable, "Number of 'drop table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ \ M(DataLakeGlueCatalogGetDatabases, "Number of 'get databases' requests to Iceberg Glue catalog.", ValueType::Number) \ M(DataLakeGlueCatalogGetDatabasesMicroseconds, "Total time of 'get databases' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index c80e25df641f..60e6dbc59c12 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -66,6 +66,14 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogGetTableMetadataMicroseconds; extern const Event DataLakeRestCatalogGetCredentials; extern const Event DataLakeRestCatalogGetCredentialsMicroseconds; + extern const Event DataLakeRestCatalogCreateNamespace; + extern const Event DataLakeRestCatalogCreateNamespaceMicroseconds; + extern const Event DataLakeRestCatalogCreateTable; + extern const Event DataLakeRestCatalogCreateTableMicroseconds; + extern const Event DataLakeRestCatalogUpdateTable; + extern const Event DataLakeRestCatalogUpdateTableMicroseconds; + extern const Event DataLakeRestCatalogDropTable; + extern const Event DataLakeRestCatalogDropTableMicroseconds; } namespace DataLake @@ -1075,6 +1083,8 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); sendRequest(endpoint, request_body); } catch (...) @@ -1117,6 +1127,8 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); sendRequest(endpoint, request_body); } catch (const DB::HTTPException & ex) @@ -1182,6 +1194,8 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); sendRequest(endpoint, request_body); } catch (const DB::HTTPException &) @@ -1205,6 +1219,8 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ Poco::JSON::Object::Ptr request_body = nullptr; try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index bf0be97754e5..e00a7de586c7 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,12 @@ namespace DB::ServerSetting extern const ServerSettingsUInt64 s3_retry_attempts; } +namespace ProfileEvents +{ + extern const Event DataLakeRestCatalogDropTable; + extern const Event DataLakeRestCatalogDropTableMicroseconds; +} + namespace DataLake { @@ -225,6 +232,8 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta Poco::JSON::Object::Ptr request_body = nullptr; try { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex)