diff --git a/.gitignore b/.gitignore index 3031a9925..e6536463e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ build/ /tests/distr /tests/work **/out/ +**/bin/ .kotlin/ \ No newline at end of file diff --git a/admin-app/build.gradle.kts b/admin-app/build.gradle.kts index ffe11a33f..682360b65 100644 --- a/admin-app/build.gradle.kts +++ b/admin-app/build.gradle.kts @@ -27,6 +27,7 @@ val testContainersVersion: String by parent!!.extra val junitJupiterVersion: String by parent!!.extra val quartzVersion: String by parent!!.extra val logbackVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -53,12 +54,15 @@ dependencies { implementation("io.ktor:ktor-serialization-kotlinx-protobuf:$ktorVersion") implementation("io.ktor:ktor-server-cors:$ktorVersion") implementation("io.ktor:ktor-server-call-logging:$ktorVersion") + implementation("io.ktor:ktor-server-metrics-micrometer:$ktorVersion") implementation("io.ktor:ktor-server-compression:$ktorVersion") implementation("io.ktor:ktor-server-resources:$ktorVersion") implementation("io.ktor:ktor-server-swagger:$ktorVersion") implementation("io.github.microutils:kotlin-logging-jvm:$microutilsLoggingVersion") implementation("org.kodein.di:kodein-di-framework-ktor-server-jvm:$kodeinVersion") implementation("ch.qos.logback:logback-classic:$logbackVersion") + implementation("io.micrometer:micrometer-core:$micrometerVersion") + implementation("io.micrometer:micrometer-registry-prometheus:$micrometerVersion") implementation("com.zaxxer:HikariCP:$zaxxerHikaricpVersion") implementation("org.postgresql:postgresql:$postgresSqlVersion") implementation("org.quartz-scheduler:quartz:$quartzVersion") diff --git a/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt b/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt index 9b11d5ee1..2b0cfba26 100644 --- a/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt +++ b/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt @@ -24,6 +24,7 @@ import com.epam.drill.admin.metrics.route.metricsRoutes import com.epam.drill.admin.common.route.commonStatusPages import com.epam.drill.admin.common.scheduler.DrillScheduler import com.epam.drill.admin.config.SchedulerConfig +import com.epam.drill.admin.config.monitoringDIModule import com.epam.drill.admin.config.schedulerDIModule import com.epam.drill.admin.etl.config.etlDIModule import com.epam.drill.admin.etl.config.updateMetricsEtlJob @@ -41,10 +42,11 @@ import io.ktor.serialization.kotlinx.protobuf.* import io.ktor.server.application.* import io.ktor.server.application.call import io.ktor.server.auth.* -import io.ktor.server.plugins.callloging.* +import io.ktor.server.plugins.calllogging.* import io.ktor.server.plugins.compression.* import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.cors.routing.* +import io.ktor.server.metrics.micrometer.* import io.ktor.server.plugins.origin import io.ktor.server.plugins.statuspages.* import io.ktor.server.plugins.swagger.* @@ -52,19 +54,27 @@ import io.ktor.server.request.* import io.ktor.server.resources.* import io.ktor.server.response.* import io.ktor.server.routing.* +import io.micrometer.core.instrument.MeterRegistry +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json +import io.micrometer.prometheus.PrometheusMeterRegistry import mu.KotlinLogging +import org.kodein.di.allInstances import org.kodein.di.instance import org.kodein.di.ktor.closestDI import org.kodein.di.ktor.di import javax.sql.DataSource private val logger = KotlinLogging.logger {} +private val loggingIgnorePaths = setOf("/metrics", "/swagger") fun Application.module() { val oauth2Enabled = oauth2Enabled val simpleAuthEnabled = simpleAuthEnabled di { + import(monitoringDIModule) import(dataSourceDIModule) import(schedulerDIModule) import(jwtServicesDIModule) @@ -92,9 +102,11 @@ fun Application.module() { if (oauth2Enabled) configureOAuthAuthentication(di) roleBasedAuthentication() } + shutdownCloseableServices() routing { rootRoute() swaggerUI(path = "swagger", swaggerFile = "openapi.yml") + metricsRoute() if (oauth2Enabled) configureOAuthRoutes() route("/api") { //UI @@ -151,6 +163,10 @@ fun Application.module() { private fun Application.installPlugins() { install(CallLogging) { + filter { call -> + val path = call.request.path() + loggingIgnorePaths.none { path.startsWith(it) } + } format { call -> val status = call.response.status() val httpMethod = call.request.httpMethod.value @@ -188,6 +204,11 @@ private fun Application.installPlugins() { exposeHeader(HttpHeaders.Authorization) exposeHeader(HttpHeaders.ContentType) } + + val meterRegistry by closestDI().instance() + install(MicrometerMetrics) { + registry = meterRegistry + } } private fun StatusPagesConfig.defaultStatusPages() { @@ -212,7 +233,7 @@ private fun Application.initScheduler() { val scheduler by closestDI().instance() scheduler.init(KodeinJobFactory(closestDI()), dataSource) - environment.monitor.subscribe(ApplicationStopped) { + monitor.subscribe(ApplicationStopped) { scheduler.shutdown() } scheduler.start() @@ -235,4 +256,25 @@ val Application.jsCoverageConverterAddress: String .propertyOrNull("jsCoverageConverterAddress") ?.getString() ?.takeIf { it.isNotBlank() } - ?: "http://localhost:8092" // TODO think of default \ No newline at end of file + ?: "http://localhost:8092" // TODO think of default + +private fun Application.shutdownCloseableServices() { + val closableComponents: List by closestDI().allInstances() + + monitor.subscribe(ApplicationStopping) { + runBlocking { + closableComponents.map { + async { + it.close() + } + }.awaitAll() + } + } +} + +private fun Route.metricsRoute() { + val meterRegistry by closestDI().instance() + get("/metrics") { + call.respondText(meterRegistry.scrape(), ContentType.Text.Plain.withCharset(Charsets.UTF_8)) + } +} \ No newline at end of file diff --git a/admin-app/src/main/kotlin/com/epam/drill/admin/config/DatabaseConfig.kt b/admin-app/src/main/kotlin/com/epam/drill/admin/config/DatabaseConfig.kt index 90f260a04..df0aaeb26 100644 --- a/admin-app/src/main/kotlin/com/epam/drill/admin/config/DatabaseConfig.kt +++ b/admin-app/src/main/kotlin/com/epam/drill/admin/config/DatabaseConfig.kt @@ -17,6 +17,8 @@ package com.epam.drill.admin.config import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource +import com.zaxxer.hikari.metrics.micrometer.MicrometerMetricsTrackerFactory +import io.micrometer.core.instrument.Metrics import io.ktor.server.application.* import io.ktor.server.config.* import org.kodein.di.DI @@ -48,8 +50,9 @@ class DatabaseConfig(private val config: ApplicationConfig) { get() = config.propertyOrNull("ssl")?.getString()?.toBooleanStrictOrNull() ?: false } -private fun DatabaseConfig.toHikariConfig(): HikariConfig = HikariConfig().apply { +private fun DatabaseConfig.toHikariConfig(poolName: String): HikariConfig = HikariConfig().apply { this.driverClassName = "org.postgresql.Driver" + this.poolName = poolName this.jdbcUrl = "jdbc:postgresql://${host}:${port}/${databaseName}" this.username = this@toHikariConfig.username this.password = this@toHikariConfig.password @@ -58,6 +61,7 @@ private fun DatabaseConfig.toHikariConfig(): HikariConfig = HikariConfig().apply this.transactionIsolation = "TRANSACTION_READ_UNCOMMITTED" this.addDataSourceProperty("rewriteBatchedInserts", true) this.addDataSourceProperty("rewriteBatchedStatements", true) + this.metricsTrackerFactory = MicrometerMetricsTrackerFactory(Metrics.globalRegistry) if (ssl) { this.addDataSourceProperty("ssl", true) this.addDataSourceProperty("sslmode", "require") @@ -73,7 +77,7 @@ val dataSourceDIModule = DI.Module("dataSource") { DatabaseConfig(instance().environment.config.config("drill.metrics.database")) } bind() with singleton { - HikariDataSource(instance().toHikariConfig()) + HikariDataSource(instance().toHikariConfig("drill-admin-main")) } bind(tag = "metrics") with singleton { val mainConfig = instance() @@ -86,7 +90,7 @@ val dataSourceDIModule = DI.Module("dataSource") { ) { instance() } else { - HikariDataSource(metricsConfig.toHikariConfig()) + HikariDataSource(metricsConfig.toHikariConfig("drill-admin-metrics")) } } } \ No newline at end of file diff --git a/admin-app/src/main/kotlin/com/epam/drill/admin/config/MonitoringModule.kt b/admin-app/src/main/kotlin/com/epam/drill/admin/config/MonitoringModule.kt new file mode 100644 index 000000000..ae69638ff --- /dev/null +++ b/admin-app/src/main/kotlin/com/epam/drill/admin/config/MonitoringModule.kt @@ -0,0 +1,61 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.config + +import io.micrometer.core.instrument.Clock +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Metrics +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics +import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics +import io.micrometer.core.instrument.binder.system.ProcessorMetrics +import io.micrometer.core.instrument.binder.system.UptimeMetrics +import io.micrometer.prometheus.PrometheusConfig +import io.micrometer.prometheus.PrometheusMeterRegistry +import io.prometheus.client.CollectorRegistry +import org.kodein.di.DI +import org.kodein.di.bind +import org.kodein.di.singleton + +val monitoringDIModule = DI.Module("monitoring") { + bind() with singleton { + prometheusRegistry + } + bind() with singleton { + prometheusRegistry + } +} + +private val prometheusRegistry: PrometheusMeterRegistry by lazy { + PrometheusMeterRegistry( + PrometheusConfig.DEFAULT, + CollectorRegistry.defaultRegistry, + Clock.SYSTEM + ).also { registry -> + if (Metrics.globalRegistry.registries.none { it === registry }) { + Metrics.addRegistry(registry) + } + ClassLoaderMetrics().bindTo(registry) + JvmMemoryMetrics().bindTo(registry) + JvmGcMetrics().bindTo(registry) + JvmThreadMetrics().bindTo(registry) + ProcessorMetrics().bindTo(registry) + FileDescriptorMetrics().bindTo(registry) + UptimeMetrics().bindTo(registry) + } +} diff --git a/admin-app/src/main/resources/application.conf b/admin-app/src/main/resources/application.conf index 4b95d5e13..b72194a17 100644 --- a/admin-app/src/main/resources/application.conf +++ b/admin-app/src/main/resources/application.conf @@ -16,6 +16,20 @@ ktor { } } +kafka { + bootstrapServers = "localhost:9092" + bootstrapServers = ${?DRILL_KAFKA_BOOTSTRAP_SERVERS} + securityProtocol = ${?DRILL_KAFKA_SECURITY_PROTOCOL} + saslMechanism = ${?DRILL_KAFKA_SASL_MECHANISM} + saslJaasConfig = ${?DRILL_KAFKA_SASL_JAAS_CONFIG} + sslTruststoreLocation = ${?DRILL_KAFKA_SSL_TRUSTSTORE_LOCATION} + sslTruststorePassword = ${?DRILL_KAFKA_SSL_TRUSTSTORE_PASSWORD} + sslKeystoreLocation = ${?DRILL_KAFKA_SSL_KEYSTORE_LOCATION} + sslKeystorePassword = ${?DRILL_KAFKA_SSL_KEYSTORE_PASSWORD} + sslKeyPassword = ${?DRILL_KAFKA_SSL_KEY_PASSWORD} + sslEndpointIdentificationAlgorithm = ${?DRILL_KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM} +} + drill { database { host = ${?DRILL_DB_HOST} @@ -72,6 +86,29 @@ drill { } } rawData { + queue { + type = ${?DRILL_RAW_DATA_QUEUE_TYPE} + capacity = ${?DRILL_RAW_DATA_QUEUE_CAPACITY} + workers = ${?DRILL_RAW_DATA_QUEUE_WORKERS} + kafka { + topic = "drill-raw-data" + topic = ${?DRILL_RAW_DATA_QUEUE_KAFKA_TOPIC} + consumerGroupId = "drill-writer" + consumerGroupId = ${?DRILL_RAW_DATA_QUEUE_KAFKA_CONSUMER_GROUP_ID} + pollTimeoutMs = 500 + pollTimeoutMs = ${?DRILL_RAW_DATA_QUEUE_KAFKA_POLL_TIMEOUT_MS} + shutdownTimeoutMs = 5000 + shutdownTimeoutMs = ${?DRILL_RAW_DATA_QUEUE_KAFKA_SHUTDOWN_TIMEOUT_MS} + producer { + clientId = "drill-admin-raw-data-producer" + clientId = ${?DRILL_RAW_DATA_QUEUE_KAFKA_PRODUCER_CLIENT_ID} + } + consumer { + clientId = "drill-admin-raw-data-consumer" + clientId = ${?DRILL_RAW_DATA_QUEUE_KAFKA_CONSUMER_CLIENT_ID} + } + } + } } metrics { database { diff --git a/admin-auth/build.gradle.kts b/admin-auth/build.gradle.kts index 1fba85677..5b25229b3 100644 --- a/admin-auth/build.gradle.kts +++ b/admin-auth/build.gradle.kts @@ -17,6 +17,7 @@ val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra val mockitoKotlinVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val jbcryptVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra val flywaydbVersion: String by parent!!.extra @@ -72,7 +73,7 @@ dependencies { testImplementation("org.testcontainers:postgresql:$testContainersVersion") testImplementation("org.postgresql:postgresql:$postgresSqlVersion") testImplementation("com.zaxxer:HikariCP:$zaxxerHikaricpVersion") - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") } tasks { diff --git a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/ApiKeyAuthenticationProvider.kt b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/ApiKeyAuthenticationProvider.kt index 4ddec036d..4b5278d15 100644 --- a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/ApiKeyAuthenticationProvider.kt +++ b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/ApiKeyAuthenticationProvider.kt @@ -40,7 +40,7 @@ class ApiKeyAuthenticationProvider internal constructor( */ class Configuration internal constructor(name: String?) : Config(name) { - internal lateinit var authenticationFunction: suspend ApplicationCall.(String) -> Principal? + internal lateinit var authenticationFunction: suspend ApplicationCall.(String) -> Any? internal var challengeFunction: suspend (ApplicationCall) -> Unit = { call -> call.respond(HttpStatusCode.Unauthorized) @@ -60,7 +60,7 @@ class ApiKeyAuthenticationProvider internal constructor( * A function that will check given API key retrieved from [headerName] and return [Principal], * or null if credential does not correspond to an authenticated principal. */ - fun validate(body: suspend ApplicationCall.(String) -> Principal?) { + fun validate(body: suspend ApplicationCall.(String) -> Any?) { authenticationFunction = body } diff --git a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/OAuth2Module.kt b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/OAuth2Module.kt index 5bb97395f..334b57bf6 100644 --- a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/OAuth2Module.kt +++ b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/OAuth2Module.kt @@ -25,7 +25,7 @@ import io.ktor.server.application.* import io.ktor.server.auth.* import io.ktor.client.* import io.ktor.client.engine.apache.* -import io.ktor.content.* +import io.ktor.http.content.* import io.ktor.http.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.response.* diff --git a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/RoleBasedAuthorization.kt b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/RoleBasedAuthorization.kt index 8bd871cea..b3432cc51 100644 --- a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/RoleBasedAuthorization.kt +++ b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/config/RoleBasedAuthorization.kt @@ -27,7 +27,7 @@ class RoleBasedAuthConfiguration { } class AuthorizedRouteSelector(private val description: String) : RouteSelector() { - override fun evaluate(context: RoutingResolveContext, segmentIndex: Int) = RouteSelectorEvaluation.Constant + override suspend fun evaluate(context: RoutingResolveContext, segmentIndex: Int) = RouteSelectorEvaluation.Constant override fun toString(): String = "(authorize ${description})" } diff --git a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/route/UserManagementRoutes.kt b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/route/UserManagementRoutes.kt index df33daada..112d208a9 100644 --- a/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/route/UserManagementRoutes.kt +++ b/admin-auth/src/main/kotlin/com/epam/drill/admin/auth/route/UserManagementRoutes.kt @@ -28,7 +28,6 @@ import io.ktor.server.application.* import io.ktor.server.auth.* import io.ktor.server.request.* import io.ktor.server.routing.* -import io.ktor.util.pipeline.* import org.kodein.di.instance import org.kodein.di.ktor.closestDI as di import com.epam.drill.admin.common.route.* @@ -149,7 +148,7 @@ fun Route.resetPasswordRoute() { } } -private fun PipelineContext.throwExceptionIfCurrentUserIs(userId: Int, message: String) { +private fun RoutingContext.throwExceptionIfCurrentUserIs(userId: Int, message: String) { if (call.principal()?.id == userId) throw ForbiddenOperationException(message) } \ No newline at end of file diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyAuthenticationProviderTest.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyAuthenticationProviderTest.kt index 80e7d06b1..e36780953 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyAuthenticationProviderTest.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyAuthenticationProviderTest.kt @@ -23,6 +23,7 @@ import io.ktor.http.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.testing.* +import io.ktor.client.request.* import kotlin.test.Test import kotlin.test.assertEquals @@ -32,41 +33,41 @@ class ApiKeyAuthenticationProviderTest { fun `given correct api key, authenticated request must succeed`() { val testApiKey = "test-api-key" - withTestApplication({ - install(Authentication) { - configureSimpleApiKey(testApiKey) - } - routing { - authenticate { - configureGetApiRoute() + testApplication { + application { + install(Authentication) { + configureSimpleApiKey(testApiKey) + } + routing { + authenticate { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, testApiKey) - }) { - assertEquals(HttpStatusCode.OK, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, testApiKey) } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `given incorrect api key, authenticated request must fail`() { - withTestApplication({ - install(Authentication) { - configureSimpleApiKey( "correct-api-key") - } - routing { - authenticate { - configureGetApiRoute() + testApplication { + application { + install(Authentication) { + configureSimpleApiKey("correct-api-key") + } + routing { + authenticate { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, "incorrect-api-key") - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, "incorrect-api-key") } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -74,24 +75,24 @@ class ApiKeyAuthenticationProviderTest { fun `given api key challenge function, authenticated request must fail with specified error`() { val testFailError = HttpStatusCode.BadRequest - withTestApplication({ - install(Authentication) { - apiKey { - validate { null } - challenge { call -> - call.respond(testFailError) + testApplication { + application { + install(Authentication) { + apiKey { + validate { null } + challenge { call -> + call.respond(testFailError) + } } } - } - routing { - authenticate { - configureGetApiRoute() + routing { + authenticate { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api")) { - assertEquals(testFailError, response.status()) - } + val response = client.get("/api") + assertEquals(testFailError, response.status) } } @@ -99,28 +100,28 @@ class ApiKeyAuthenticationProviderTest { fun `given custom api key header, authenticated request must succeed in retrieving api key from that header`() { val testHeader = "X-Test-Api-Key" val testApiKey = "test-api-key" - withTestApplication({ - install(Authentication) { - apiKey { - headerName = testHeader - validate { apiKey -> - apiKey - .takeIf { it == testApiKey } - ?.let { UserIdPrincipal("username") } + testApplication { + application { + install(Authentication) { + apiKey { + headerName = testHeader + validate { apiKey -> + apiKey + .takeIf { it == testApiKey } + ?.let { UserIdPrincipal("username") } + } } } - } - routing { - authenticate { - configureGetApiRoute() + routing { + authenticate { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(testHeader, testApiKey) - }) { - assertEquals(HttpStatusCode.OK, response.status()) + val response = client.get("/api") { + header(testHeader, testApiKey) } + assertEquals(HttpStatusCode.OK, response.status) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyManagementTests.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyManagementTests.kt index ce2e5f4a4..5faae357e 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyManagementTests.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyManagementTests.kt @@ -32,6 +32,7 @@ import io.ktor.server.plugins.statuspages.* import io.ktor.server.resources.* import io.ktor.server.testing.* import kotlinx.serialization.builtins.ListSerializer +import io.ktor.client.request.* import org.kodein.di.bind import org.kodein.di.ktor.di import org.kodein.di.provider @@ -70,30 +71,30 @@ class ApiKeyManagementTests { ) ) - withTestApplication(withRoute { getAllApiKeysRoute() }) { - with(handleRequest(HttpMethod.Get, "/keys") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: List = assertResponseNotNull(ListSerializer(ApiKeyView.serializer())) - assertEquals(2, response.size) + testApplication { + application(withRoute { getAllApiKeysRoute() }) + val response = client.get("/keys") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + val body: List = response.assertResponseNotNull(ListSerializer(ApiKeyView.serializer())) + assertEquals(2, body.size) } } @Test fun `given api key identifier, 'DELETE keys {id}' must delete that api key from repository`() { - withTestApplication(withRoute { deleteApiKeyRoute() }) { - with(handleRequest(HttpMethod.Delete, "/keys/1") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(apiKeyRepository) { deleteById(1) } + testApplication { + application(withRoute { deleteApiKeyRoute() }) + val response = client.delete("/keys/1") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(apiKeyRepository) { deleteById(1) } } } - private fun withRoute(route: Routing.() -> Unit): Application.() -> Unit = { + private fun withRoute(route: Route.() -> Unit): Application.() -> Unit = { install(Resources) install(ContentNegotiation) { json() diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyModuleTest.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyModuleTest.kt index a783bc5b7..0676d00c5 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyModuleTest.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/ApiKeyModuleTest.kt @@ -31,6 +31,7 @@ import io.ktor.serialization.kotlinx.json.* import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.testing.* +import io.ktor.client.request.* import org.kodein.di.* import org.kodein.di.ktor.closestDI import org.kodein.di.ktor.di @@ -72,41 +73,35 @@ class ApiKeyModuleTest { wheneverBlocking(mockApiKeyRepository) { findById(123) }.doReturn(createTestApiKeyEntity()) whenever(mockPasswordService.matchPasswords(eq("key-secret"), any())).doReturn(true) - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, validApiKey) - }) { - assertEquals(HttpStatusCode.OK, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, validApiKey) } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `without api key, api-key authenticated request must fail with 401 status`() { - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - //no adding API key header - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) - } + val response = client.get("/api") + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -117,21 +112,19 @@ class ApiKeyModuleTest { wheneverBlocking(mockApiKeyRepository) { findById(123) }.doReturn(createTestApiKeyEntity(apiKeyHash = "hash")) whenever(mockPasswordService.matchPasswords("wrong-secret", "hash")).doReturn(false) - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, invalidApiKey) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, invalidApiKey) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -143,21 +136,19 @@ class ApiKeyModuleTest { .doReturn(createTestApiKeyEntity(expiresAt = LocalDateTime.now().minusDays(1))) whenever(mockPasswordService.matchPasswords(eq("key-secret"), any())).doReturn(true) - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, expiredApiKey) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, expiredApiKey) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -169,21 +160,19 @@ class ApiKeyModuleTest { .doReturn(createTestApiKeyEntity(user = createTestUserEntity(blocked = true))) whenever(mockPasswordService.matchPasswords(eq("key-secret"), any())).doReturn(true) - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, blockedApiKey) - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, blockedApiKey) } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @@ -195,21 +184,19 @@ class ApiKeyModuleTest { .doReturn(createTestApiKeyEntity(user = createTestUserEntity(role = Role.UNDEFINED))) whenever(mockPasswordService.matchPasswords(eq("key-secret"), any())).doReturn(true) - withTestApplication({ - withApiKeyModule { - configureMocks() - } - routing { - authenticate("api-key") { - configureGetApiRoute() + testApplication { + application { + withApiKeyModule { configureMocks() } + routing { + authenticate("api-key") { + configureGetApiRoute() + } } } - }) { - with(handleRequest(HttpMethod.Get, "/api") { - addHeader(API_KEY_HEADER, undefinedRoleApiKey) - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) + val response = client.get("/api") { + header(API_KEY_HEADER, undefinedRoleApiKey) } + assertEquals(HttpStatusCode.Forbidden, response.status) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/OAuthModuleTest.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/OAuthModuleTest.kt index d3b2ee29f..ca61229c8 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/OAuthModuleTest.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/OAuthModuleTest.kt @@ -29,6 +29,8 @@ import io.ktor.server.plugins.statuspages.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.testing.* +import io.ktor.client.request.* +import io.ktor.client.statement.* import org.kodein.di.DI import org.kodein.di.bind import org.kodein.di.ktor.closestDI @@ -65,59 +67,60 @@ class OAuthModuleTest { val testIssuer = "test-issuer" val testSecret = "secret" - withTestApplication({ - environment { - put("drill.auth.jwt.issuer", testIssuer) - put("drill.auth.jwt.secret", testSecret) - } - withTestOAuthModule() - routing { - authenticate("jwt") { - get("/protected") { - call.respond(HttpStatusCode.OK) + testApplication { + application { + environment { + put("drill.auth.jwt.issuer", testIssuer) + put("drill.auth.jwt.secret", testSecret) + } + withTestOAuthModule() + routing { + authenticate("jwt") { + get("/protected") { + call.respond(HttpStatusCode.OK) + } } } } - }) { - with(handleRequest(HttpMethod.Get, "/protected") { + val response = client.get("/protected") { addJwtToken( username = testUsername, issuer = testIssuer, secret = testSecret ) { - addHeader("Cookie", "$JWT_COOKIE=$it") + header("Cookie", "$JWT_COOKIE=$it") } - }) { - assertEquals(HttpStatusCode.OK, response.status()) } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `oauth login request must be redirected to oauth2 authorize url`() { - withTestApplication({ - environment { - put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") - put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") - put("drill.auth.oauth2.clientId", testClientId) - put("drill.auth.oauth2.clientSecret", testClientSecret) - put("drill.auth.oauth2.scopes", "scope1, scope2") - put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") + testApplication { + application { + environment { + put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") + put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") + put("drill.auth.oauth2.clientId", testClientId) + put("drill.auth.oauth2.clientSecret", testClientSecret) + put("drill.auth.oauth2.scopes", "scope1, scope2") + put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") + } + withTestOAuthModule() } - withTestOAuthModule() - }) { - with(handleRequest(HttpMethod.Get, "/oauth/login")) { - assertEquals(HttpStatusCode.Found, response.status()) + val noRedirectClient = createClient { followRedirects = false } + val response = noRedirectClient.get("/oauth/login") + assertEquals(HttpStatusCode.Found, response.status) - val redirectedUrl = URL(response.headers[HttpHeaders.Location]) - val queryParams = redirectedUrl.queryParams() - assertEquals(testOAuthServerHost, redirectedUrl.host) - assertEquals("/authorizeUrl", redirectedUrl.path) - assertEquals(testClientId, queryParams["client_id"]) - assertEquals("http://$testDrillHost/oauth/callback", queryParams["redirect_uri"]) - assertEquals("code", queryParams["response_type"]) - assertNotNull(queryParams["state"]) - } + val redirectedUrl = URL(response.headers[HttpHeaders.Location]) + val queryParams = redirectedUrl.queryParams() + assertEquals(testOAuthServerHost, redirectedUrl.host) + assertEquals("/authorizeUrl", redirectedUrl.path) + assertEquals(testClientId, queryParams["client_id"]) + assertEquals("http://$testDrillHost/oauth/callback", queryParams["redirect_uri"]) + assertEquals("code", queryParams["response_type"]) + assertNotNull(queryParams["state"]) } } @@ -128,53 +131,53 @@ class OAuthModuleTest { val testAuthenticationCode = "test-code" val testState = "test-state" - withTestApplication({ - environment { - put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") - put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") - put("drill.auth.oauth2.clientId", testClientId) - put("drill.auth.oauth2.clientSecret", testClientSecret) - put("drill.auth.jwt.issuer", testIssuer) - put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") - } - withTestOAuthModule { - bind(overrides = true) with provider { mockOAuthService } - mockHttpClient("oauthHttpClient", - "/accessTokenUrl" shouldRespond { request -> - request.formData().apply { - assertEquals(testClientId, this["client_id"]) - assertEquals(testClientSecret, this["client_secret"]) - assertEquals(testAuthenticationCode, this["code"]) - assertEquals(testState, this["state"]) - } - respondOk( - """ - { - "access_token":"test-access-token", - "refresh_token":"test-refresh-token" + testApplication { + application { + environment { + put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") + put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") + put("drill.auth.oauth2.clientId", testClientId) + put("drill.auth.oauth2.clientSecret", testClientSecret) + put("drill.auth.jwt.issuer", testIssuer) + put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") + } + withTestOAuthModule { + bind(overrides = true) with provider { mockOAuthService } + mockHttpClient("oauthHttpClient", + "/accessTokenUrl" shouldRespond { request -> + request.formData().apply { + assertEquals(testClientId, this["client_id"]) + assertEquals(testClientSecret, this["client_secret"]) + assertEquals(testAuthenticationCode, this["code"]) + assertEquals(testState, this["state"]) } - """.trimIndent() - ) - } - ) + respondOk( + """ + { + "access_token":"test-access-token", + "refresh_token":"test-refresh-token" + } + """.trimIndent() + ) + } + ) + } } - }) { wheneverBlocking(mockOAuthService) { signInThroughOAuth(any()) }.thenReturn(UserInfoView(id = 123, testUsername, Role.USER, false)) - with(handleRequest(HttpMethod.Get, "/oauth/callback?code=$testAuthenticationCode&state=$testState")) { - assertEquals(HttpStatusCode.Found, response.status()) - assertEquals("http://$testDrillHost/drill", response.headers[HttpHeaders.Location]) - assertNotNull(response.cookies[JWT_COOKIE]).let { jwtCookie -> - assertNotNull(jwtCookie.value) - JWT.decode(jwtCookie.value).apply { - assertEquals(testUsername, subject) - assertEquals(testIssuer, issuer) - assertEquals(Role.USER.name, getClaim("role").asString()) - } - assertTrue(jwtCookie.httpOnly) - assertEquals("/", jwtCookie.path) - } + val noRedirectClient = createClient { followRedirects = false } + val response = noRedirectClient.get("/oauth/callback?code=$testAuthenticationCode&state=$testState") + assertEquals(HttpStatusCode.Found, response.status) + assertEquals("http://$testDrillHost/drill", response.headers[HttpHeaders.Location]) + val jwtCookie = assertNotNull(response.setCookie().find { it.name == JWT_COOKIE }) + assertNotNull(jwtCookie.value) + JWT.decode(jwtCookie.value).apply { + assertEquals(testUsername, subject) + assertEquals(testIssuer, issuer) + assertEquals(Role.USER.name, getClaim("role").asString()) } + assertTrue(jwtCookie.httpOnly) + assertEquals("/", jwtCookie.path) } } @@ -183,26 +186,26 @@ class OAuthModuleTest { val wrongAuthenticationCode = "invalid-code" val testState = "test-state" - withTestApplication({ - environment { - put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") - put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") - put("drill.auth.oauth2.clientId", testClientId) - put("drill.auth.oauth2.clientSecret", testClientSecret) - put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") - } - withTestOAuthModule { - bind(overrides = true) with provider { mockOAuthService } - mockHttpClient("oauthHttpClient", - "/accessTokenUrl" shouldRespond { - respondError(HttpStatusCode.Unauthorized, "Invalid authentication code") - } - ) - } - }) { - with(handleRequest(HttpMethod.Get, "/oauth/callback?code=$wrongAuthenticationCode&state=$testState")) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) + testApplication { + application { + environment { + put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") + put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") + put("drill.auth.oauth2.clientId", testClientId) + put("drill.auth.oauth2.clientSecret", testClientSecret) + put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") + } + withTestOAuthModule { + bind(overrides = true) with provider { mockOAuthService } + mockHttpClient("oauthHttpClient", + "/accessTokenUrl" shouldRespond { + respondError(HttpStatusCode.Unauthorized, "Invalid authentication code") + } + ) + } } + val response = client.get("/oauth/callback?code=$wrongAuthenticationCode&state=$testState") + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -211,34 +214,34 @@ class OAuthModuleTest { val testAuthenticationCode = "test-code" val testState = "test-state" - withTestApplication({ - environment { - put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") - put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") - put("drill.auth.oauth2.clientId", testClientId) - put("drill.auth.oauth2.clientSecret", testClientSecret) - put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") - } - withTestOAuthModule { - bind(overrides = true) with provider { mockOAuthService } - mockHttpClient("oauthHttpClient", - "/accessTokenUrl" shouldRespond { request -> - respondOk( - """ - { - "access_token":"test-access-token", - "refresh_token":"test-refresh-token" - } - """.trimIndent() - ) - } - ) + testApplication { + application { + environment { + put("drill.auth.oauth2.authorizeUrl", "http://$testOAuthServerHost/authorizeUrl") + put("drill.auth.oauth2.accessTokenUrl", "http://$testOAuthServerHost/accessTokenUrl") + put("drill.auth.oauth2.clientId", testClientId) + put("drill.auth.oauth2.clientSecret", testClientSecret) + put("drill.auth.oauth2.redirectUrl", "http://$testDrillHost/drill") + } + withTestOAuthModule { + bind(overrides = true) with provider { mockOAuthService } + mockHttpClient("oauthHttpClient", + "/accessTokenUrl" shouldRespond { request -> + respondOk( + """ + { + "access_token":"test-access-token", + "refresh_token":"test-refresh-token" + } + """.trimIndent() + ) + } + ) + } } - }) { wheneverBlocking(mockOAuthService) { signInThroughOAuth(any()) }.thenThrow(OAuthUnauthorizedException()) - with(handleRequest(HttpMethod.Get, "/oauth/callback?code=$testAuthenticationCode&state=$testState")) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) - } + val response = client.get("/oauth/callback?code=$testAuthenticationCode&state=$testState") + assertEquals(HttpStatusCode.Unauthorized, response.status) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/RoleBasedAuthorizationTest.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/RoleBasedAuthorizationTest.kt index b15580f42..0374d77e6 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/RoleBasedAuthorizationTest.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/RoleBasedAuthorizationTest.kt @@ -29,84 +29,71 @@ import io.ktor.http.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.testing.* +import io.ktor.client.request.* import kotlin.test.* class RoleBasedAuthorizationTest { @Test fun `given user with admin role, request only-admins should return 200 OK`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/only-admins") { - addBasicAuth("admin", "secret") - }) { - assertEquals(HttpStatusCode.OK, response.status()) - } + testApplication { + application(config) + val response = client.get("/only-admins") { addBasicAuth("admin", "secret") } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `given user with user role, request only-admins should return 403 Access denied`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/only-admins") { - addBasicAuth("user", "secret") - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) - } + testApplication { + application(config) + val response = client.get("/only-admins") { addBasicAuth("user", "secret") } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @Test fun `given user with user role, request only-users should return 200 OK`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/only-users") { - addBasicAuth("user", "secret") - }) { - assertEquals(HttpStatusCode.OK, response.status()) - } + testApplication { + application(config) + val response = client.get("/only-users") { addBasicAuth("user", "secret") } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `given user with admin role, request only-users should return 403 Access denied`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/only-users") { - addBasicAuth("admin", "secret") - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) - } + testApplication { + application(config) + val response = client.get("/only-users") { addBasicAuth("admin", "secret") } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @Test fun `given user with user role, request admins-or-users should return 200 OK`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/admins-or-users") { - addBasicAuth("user", "secret") - }) { - assertEquals(HttpStatusCode.OK, response.status()) - } + testApplication { + application(config) + val response = client.get("/admins-or-users") { addBasicAuth("user", "secret") } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `given guest without role, request admins-or-users should return 403 Access denied`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/admins-or-users") { - addBasicAuth("guest", "secret") - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) - } + testApplication { + application(config) + val response = client.get("/admins-or-users") { addBasicAuth("guest", "secret") } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @Test fun `given guest without role, request all should return 200 OK`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/all") { - addBasicAuth("guest", "secret") - }) { - assertEquals(HttpStatusCode.OK, response.status()) - } + testApplication { + application(config) + val response = client.get("/all") { addBasicAuth("guest", "secret") } + assertEquals(HttpStatusCode.OK, response.status) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/SimpleAuthModuleTest.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/SimpleAuthModuleTest.kt index ce6c9f472..2f3f77af4 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/SimpleAuthModuleTest.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/SimpleAuthModuleTest.kt @@ -25,6 +25,7 @@ import io.ktor.http.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.testing.* +import io.ktor.client.request.* import org.kodein.di.* import org.kodein.di.ktor.closestDI import org.kodein.di.ktor.di @@ -48,57 +49,55 @@ class SimpleAuthModuleTest { @Test fun `given user with valid jwt token, request jwt-only must succeed`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/jwt-only") { + testApplication { + application(config) + val response = client.get("/jwt-only") { addJwtToken( username = "admin", issuer = testIssuer, secret = testSecret, - configureHeader = { addHeader(HttpHeaders.Cookie, "jwt=$it;") } + configureHeader = { header(HttpHeaders.Cookie, "jwt=$it;") } ) - }) { - assertEquals(HttpStatusCode.OK, response.status()) } + assertEquals(HttpStatusCode.OK, response.status) } } @Test fun `given user without jwt token, request jwt-only must fail with 401 status`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/jwt-only") { - //not to add jwt token - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) - } + testApplication { + application(config) + val response = client.get("/jwt-only") + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @Test fun `given user with expired jwt token, request jwt-only must fail with 401 status`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/jwt-only") { + testApplication { + application(config) + val response = client.get("/jwt-only") { addJwtToken( username = "admin", secret = testSecret, expiresAt = Date(System.currentTimeMillis() - 1000) ) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @Test fun `given user with invalid jwt token, request jwt-only must fail with 401 status`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/jwt-only") { + testApplication { + application(config) + val response = client.get("/jwt-only") { addJwtToken( username = "admin", secret = "wrong_secret" ) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/TestUtils.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/TestUtils.kt index 2e9470eee..04e304e43 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/TestUtils.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/TestUtils.kt @@ -34,6 +34,7 @@ import io.ktor.server.auth.jwt.* import io.ktor.client.* import io.ktor.client.engine.mock.* import io.ktor.client.request.* +import io.ktor.client.statement.* import io.ktor.server.config.* import io.ktor.http.* import io.ktor.server.testing.* @@ -56,17 +57,17 @@ import java.net.URL import java.net.URLDecoder import java.nio.charset.StandardCharsets import java.time.LocalDateTime -import java.util.* +import java.util.*; import kotlin.test.assertNotNull const val TEST_JWT_SECRET = "secret" -fun TestApplicationRequest.addBasicAuth(username: String, password: String) { +fun HttpRequestBuilder.addBasicAuth(username: String, password: String) { val encodedCredentials = String(Base64.getEncoder().encode("$username:$password".toByteArray())) - addHeader(HttpHeaders.Authorization, "Basic $encodedCredentials") + header(HttpHeaders.Authorization, "Basic $encodedCredentials") } -fun TestApplicationRequest.addJwtToken( +fun HttpRequestBuilder.addJwtToken( username: String, secret: String = TEST_JWT_SECRET, expiresAt: Date = Date(System.currentTimeMillis() + 10_000), @@ -78,7 +79,7 @@ fun TestApplicationRequest.addJwtToken( configureJwt: JWTCreator.Builder.() -> Unit = { withClaim(CLAIM_ROLE, role).withClaim(CLAIM_USER_ID, userId) }, - configureHeader: TestApplicationRequest.(String) -> Unit = { addHeader(HttpHeaders.Authorization, "Bearer $it") } + configureHeader: HttpRequestBuilder.(String) -> Unit = { header(HttpHeaders.Authorization, "Bearer $it") } ) { val token = JWT.create() .withSubject(username) @@ -90,8 +91,8 @@ fun TestApplicationRequest.addJwtToken( configureHeader(token) } -fun TestApplicationCall.assertResponseNotNull(serializer: KSerializer): T { - val value = assertNotNull(response.content) +suspend fun HttpResponse.assertResponseNotNull(serializer: KSerializer): T { + val value = assertNotNull(bodyAsText()) val response = Json.decodeFromString(DataResponse.serializer(serializer), value) return response.data } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserApiKeyTests.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserApiKeyTests.kt index 8d9614b78..8339117e5 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserApiKeyTests.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserApiKeyTests.kt @@ -34,6 +34,7 @@ import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.resources.* import io.ktor.server.testing.* +import io.ktor.client.request.* import kotlinx.datetime.toKotlinLocalDateTime import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.json.Json @@ -85,19 +86,19 @@ class UserApiKeyTests { ) ) - withTestApplication(withRoute { - authenticate { - getUserApiKeysRoute() - } - }) { - with(handleRequest(HttpMethod.Get, "/user-keys") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + getUserApiKeysRoute() + } + }) + val response = client.get("/user-keys") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addJwtToken(username = "test-user", userId = testUserId) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: List = assertResponseNotNull(ListSerializer(UserApiKeyView.serializer())) - assertEquals(2, response.size) } + assertEquals(HttpStatusCode.OK, response.status) + val body: List = response.assertResponseNotNull(ListSerializer(UserApiKeyView.serializer())) + assertEquals(2, body.size) } } @@ -112,26 +113,26 @@ class UserApiKeyTests { wheneverBlocking(secretGenerator) { generate() }.thenReturn("secret") wheneverBlocking(apiKeyBuilder) { format(any()) }.thenReturn(testApiKey) - withTestApplication(withRoute { - authenticate { - generateUserApiKeyRoute() - } - }) { - with(handleRequest(HttpMethod.Post, "/user-keys") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + generateUserApiKeyRoute() + } + }) + val response = client.post("/user-keys") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = GenerateApiKeyPayload( description = "for some client", expiryPeriod = ExpiryPeriod.ONE_MONTH ) setBody(Json.encodeToString(GenerateApiKeyPayload.serializer(), form)) addJwtToken(username = "test-user", userId = testUserId) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: ApiKeyCredentialsView = assertResponseNotNull(ApiKeyCredentialsView.serializer()) - assertEquals(testApiKeyId, response.id) - assertEquals(testApiKey, response.apiKey) - assertNotNull(response.expiresAt) } + assertEquals(HttpStatusCode.OK, response.status) + val body: ApiKeyCredentialsView = response.assertResponseNotNull(ApiKeyCredentialsView.serializer()) + assertEquals(testApiKeyId, body.id) + assertEquals(testApiKey, body.apiKey) + assertNotNull(body.expiresAt) } } @@ -143,24 +144,24 @@ class UserApiKeyTests { wheneverBlocking(secretGenerator) { generate() }.thenReturn("secret") wheneverBlocking(apiKeyBuilder) { format(any()) }.thenReturn("test-api-key") - withTestApplication(withRoute { - authenticate { - generateUserApiKeyRoute() - } - }) { - with(handleRequest(HttpMethod.Post, "/user-keys") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + generateUserApiKeyRoute() + } + }) + val response = client.post("/user-keys") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = GenerateApiKeyPayload( description = "for some client", expiryPeriod = ExpiryPeriod.THREE_MONTHS ) setBody(Json.encodeToString(GenerateApiKeyPayload.serializer(), form)) addJwtToken(username = "test-user", userId = 43) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: ApiKeyCredentialsView = assertResponseNotNull(ApiKeyCredentialsView.serializer()) - assertEquals(firstJanuary2023.plusMonths(3).toKotlinLocalDateTime(), response.expiresAt) } + assertEquals(HttpStatusCode.OK, response.status) + val body: ApiKeyCredentialsView = response.assertResponseNotNull(ApiKeyCredentialsView.serializer()) + assertEquals(firstJanuary2023.plusMonths(3).toKotlinLocalDateTime(), body.expiresAt) } } @@ -174,18 +175,18 @@ class UserApiKeyTests { user = UserEntity(id = testUserId, username = "test-user", role = "USER") ) ) - withTestApplication(withRoute { - authenticate { - deleteUserApiKeyRoute() - } - }) { - with(handleRequest(HttpMethod.Delete, "/user-keys/$testApiKey") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + deleteUserApiKeyRoute() + } + }) + val response = client.delete("/user-keys/$testApiKey") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addJwtToken(username = "test-user", userId = testUserId) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(apiKeyRepository) { deleteById(testApiKey) } } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(apiKeyRepository) { deleteById(testApiKey) } } } @@ -200,21 +201,21 @@ class UserApiKeyTests { user = UserEntity(id = anotherUserId, username = "another-user", role = "USER") ) ) - withTestApplication(withRoute { - authenticate { - deleteUserApiKeyRoute() - } - }) { - with(handleRequest(HttpMethod.Delete, "/user-keys/$testApiKey") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + deleteUserApiKeyRoute() + } + }) + val response = client.delete("/user-keys/$testApiKey") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addJwtToken(username = "test-user", userId = testUserId) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } - private fun withRoute(route: Routing.() -> Unit): Application.() -> Unit = { + private fun withRoute(route: Route.() -> Unit): Application.() -> Unit = { install(Resources) install(ContentNegotiation) { json() diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserAuthenticationTests.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserAuthenticationTests.kt index 6cc4ffc4c..8dc13cb7a 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserAuthenticationTests.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserAuthenticationTests.kt @@ -39,6 +39,8 @@ import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.resources.* import io.ktor.server.testing.* +import io.ktor.client.request.* +import io.ktor.client.statement.* import kotlinx.serialization.json.Json import org.kodein.di.bind import org.kodein.di.eagerSingleton @@ -82,16 +84,16 @@ class UserAuthenticationTest { .thenReturn(true) whenever(tokenService.issueToken(any())).thenReturn("token") - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-in") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-in") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val payload = LoginPayload(username = testUsername, password = "secret") setBody(Json.encodeToString(LoginPayload.serializer(), payload)) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response = assertResponseNotNull(TokenView.serializer()) - assertEquals("token", response.token) } + assertEquals(HttpStatusCode.OK, response.status) + val body = response.assertResponseNotNull(TokenView.serializer()) + assertEquals("token", body.token) } } @@ -102,22 +104,22 @@ class UserAuthenticationTest { whenever(passwordService.hashPassword("secret")).thenReturn("hash") wheneverBlocking(userRepository) { create(any()) }.thenAnswer(CopyUserWithID) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-up") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-up") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = RegistrationPayload(username = testUsername, password = "secret") setBody(Json.encodeToString(RegistrationPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { - create( - UserEntity( - username = testUsername, - passwordHash = "hash", - role = Role.UNDEFINED.name - ) + } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { + create( + UserEntity( + username = testUsername, + passwordHash = "hash", + role = Role.UNDEFINED.name ) - } + ) } } } @@ -128,16 +130,16 @@ class UserAuthenticationTest { wheneverBlocking(userRepository) { findByUsername(testUsername) } .thenReturn(UserEntity(id = 1, username = testUsername, passwordHash = "hash", role = "USER")) - withTestApplication(config) { - with(handleRequest(HttpMethod.Get, "/user-info") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.get("/user-info") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addBasicAuth(testUsername, "secret") - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val userInfo = assertResponseNotNull(UserInfoView.serializer()) - assertEquals(testUsername, userInfo.username) - assertEquals(Role.USER, userInfo.role) } + assertEquals(HttpStatusCode.OK, response.status) + val userInfo = response.assertResponseNotNull(UserInfoView.serializer()) + assertEquals(testUsername, userInfo.username) + assertEquals(Role.USER, userInfo.role) } } @@ -150,16 +152,16 @@ class UserAuthenticationTest { whenever(passwordService.hashPassword("secret2")) .thenReturn("hash2") - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/update-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/update-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addBasicAuth("guest", "secret") val form = ChangePasswordPayload(oldPassword = "secret", newPassword = "secret2") setBody(Json.encodeToString(ChangePasswordPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { update(USER_GUEST.copy(passwordHash = "hash2")) } } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { update(USER_GUEST.copy(passwordHash = "hash2")) } } } @@ -168,14 +170,14 @@ class UserAuthenticationTest { wheneverBlocking(userRepository) { findByUsername("unknown") } .thenReturn(null) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-in") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-in") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = LoginPayload(username = "unknown", password = "secret") setBody(Json.encodeToString(LoginPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -186,14 +188,14 @@ class UserAuthenticationTest { whenever(passwordService.matchPasswords("incorrect", "hash")) .thenReturn(false) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-in") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-in") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = LoginPayload(username = "guest", password = "incorrect") setBody(Json.encodeToString(LoginPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -208,14 +210,14 @@ class UserAuthenticationTest { whenever(passwordService.matchPasswords("secret", "hash")) .thenReturn(true) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-in") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-in") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = LoginPayload(username = "blocked_user", password = "secret") setBody(Json.encodeToString(LoginPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @@ -230,14 +232,14 @@ class UserAuthenticationTest { whenever(passwordService.matchPasswords("secret", "hash")) .thenReturn(true) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-in") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-in") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = LoginPayload(username = "undefined_role", password = "secret") setBody(Json.encodeToString(LoginPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.Forbidden, response.status()) } + assertEquals(HttpStatusCode.Forbidden, response.status) } } @@ -246,28 +248,27 @@ class UserAuthenticationTest { wheneverBlocking(userRepository) { findByUsername("guest") } .thenReturn(USER_GUEST) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-up") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/sign-up") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = RegistrationPayload(username = "guest", password = "secret") setBody(Json.encodeToString(RegistrationPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.BadRequest, response.status()) } + assertEquals(HttpStatusCode.BadRequest, response.status) } } @Test fun `without authentication 'POST update-password' must fail with 401 status`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/update-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - //not add auth + testApplication { + application(config) + val response = client.post("/update-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = ChangePasswordPayload(oldPassword = "secret", newPassword = "secret2") setBody(Json.encodeToString(ChangePasswordPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.Unauthorized, response.status()) } + assertEquals(HttpStatusCode.Unauthorized, response.status) } } @@ -278,15 +279,15 @@ class UserAuthenticationTest { whenever(passwordService.matchPasswords("incorrect", "hash")) .thenReturn(false) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/update-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/update-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addBasicAuth("guest", "secret") val form = ChangePasswordPayload(oldPassword = "incorrect", newPassword = "secret2") setBody(Json.encodeToString(ChangePasswordPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.BadRequest, response.status()) } + assertEquals(HttpStatusCode.BadRequest, response.status) } } @@ -294,29 +295,28 @@ class UserAuthenticationTest { fun `given external user 'POST update-password' must fail with 422 status`() { wheneverBlocking(userRepository) { findByUsername("external-user") } .thenReturn( - //null password hash means the user is external UserEntity(id = 123, username = "external-user", passwordHash = null, role = "USER") ) - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/update-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(config) + val response = client.post("/update-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addBasicAuth("external-user", "") val form = ChangePasswordPayload(oldPassword = "", newPassword = "secret") setBody(Json.encodeToString(ChangePasswordPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } @Test fun `'POST sign-out' must clear jwt cookie`() { - withTestApplication(config) { - with(handleRequest(HttpMethod.Post, "/sign-out")) { - assertEquals(HttpStatusCode.OK, response.status()) - assertTrue(response.cookies[JWT_COOKIE]?.value.isNullOrEmpty()) - } + testApplication { + application(config) + val response = client.post("/sign-out") + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.setCookie().find { it.name == JWT_COOKIE }?.value.isNullOrEmpty()) } } diff --git a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserManagementTests.kt b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserManagementTests.kt index 62102d17d..287ccae30 100644 --- a/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserManagementTests.kt +++ b/admin-auth/src/test/kotlin/com/epam/drill/admin/auth/UserManagementTests.kt @@ -34,6 +34,7 @@ import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.resources.* import io.ktor.server.testing.* +import io.ktor.client.request.* import kotlinx.serialization.builtins.ListSerializer import kotlinx.serialization.json.Json import org.kodein.di.bind @@ -76,14 +77,14 @@ class UserManagementTest { wheneverBlocking(userRepository) { findAll() } .thenReturn(listOf(USER_ADMIN, USER_USER)) - withTestApplication(withRoute { getUsersRoute() }) { - with(handleRequest(HttpMethod.Get, "/users") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: List = assertResponseNotNull(ListSerializer(UserView.serializer())) - assertEquals(2, response.size) + testApplication { + application(withRoute { getUsersRoute() }) + val response = client.get("/users") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + val body: List = response.assertResponseNotNull(ListSerializer(UserView.serializer())) + assertEquals(2, body.size) } } @@ -97,14 +98,14 @@ class UserManagementTest { ) ) - withTestApplication(withRoute { getUserRoute() }) { - with(handleRequest(HttpMethod.Get, "/users/1") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - val response: UserView = assertResponseNotNull(UserView.serializer()) - assertEquals("admin", response.username) + testApplication { + application(withRoute { getUserRoute() }) + val response = client.get("/users/1") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + val body: UserView = response.assertResponseNotNull(UserView.serializer()) + assertEquals("admin", body.username) } } @@ -115,17 +116,17 @@ class UserManagementTest { wheneverBlocking(userRepository) { update(any()) } .thenAnswer(CopyUser) - withTestApplication(withRoute { editUserRoute() }) { - with(handleRequest(HttpMethod.Put, "/users/1") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { editUserRoute() }) + val response = client.put("/users/1") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = EditUserPayload(role = Role.USER) setBody(Json.encodeToString(EditUserPayload.serializer(), form)) - }) { - verifyBlocking(userRepository) { update(USER_ADMIN.copy(role = "USER")) } - assertEquals(HttpStatusCode.OK, response.status()) - val response: UserView = assertResponseNotNull(UserView.serializer()) - assertEquals(Role.USER, response.role) } + verifyBlocking(userRepository) { update(USER_ADMIN.copy(role = "USER")) } + assertEquals(HttpStatusCode.OK, response.status) + val body: UserView = response.assertResponseNotNull(UserView.serializer()) + assertEquals(Role.USER, body.role) } } @@ -134,13 +135,13 @@ class UserManagementTest { wheneverBlocking(userRepository) { findById(1) } .thenReturn(USER_ADMIN) - withTestApplication(withRoute { deleteUserRoute() }) { - with(handleRequest(HttpMethod.Delete, "/users/1") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { deleteById(1) } + testApplication { + application(withRoute { deleteUserRoute() }) + val response = client.delete("/users/1") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { deleteById(1) } } } @@ -149,13 +150,13 @@ class UserManagementTest { wheneverBlocking(userRepository) { findById(1) } .thenReturn(USER_ADMIN) - withTestApplication(withRoute { blockUserRoute() }) { - with(handleRequest(HttpMethod.Patch, "/users/1/block") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { update(USER_ADMIN.copy(blocked = true)) } + testApplication { + application(withRoute { blockUserRoute() }) + val response = client.patch("/users/1/block") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { update(USER_ADMIN.copy(blocked = true)) } } } @@ -164,13 +165,13 @@ class UserManagementTest { wheneverBlocking(userRepository) { findById(1) } .thenReturn(USER_ADMIN.copy(blocked = true)) - withTestApplication(withRoute { unblockUserRoute() }) { - with(handleRequest(HttpMethod.Patch, "/users/1/unblock") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { update(USER_ADMIN.copy(blocked = false)) } + testApplication { + application(withRoute { unblockUserRoute() }) + val response = client.patch("/users/1/unblock") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { update(USER_ADMIN.copy(blocked = false)) } } } @@ -183,63 +184,63 @@ class UserManagementTest { whenever(passwordService.hashPassword("newsecret")) .thenReturn("newhash") - withTestApplication(withRoute { resetPasswordRoute() }) { - with(handleRequest(HttpMethod.Patch, "/users/1/reset-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.OK, response.status()) - verifyBlocking(userRepository) { update(USER_ADMIN.copy(passwordHash = "newhash")) } + testApplication { + application(withRoute { resetPasswordRoute() }) + val response = client.patch("/users/1/reset-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.OK, response.status) + verifyBlocking(userRepository) { update(USER_ADMIN.copy(passwordHash = "newhash")) } } } @Test fun `given user identifier equal to current user, 'DELETE users {id}' must fail`() { - withTestApplication(withRoute { - authenticate { - deleteUserRoute() - } - }) { - with(handleRequest(HttpMethod.Delete, "/users/123") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + deleteUserRoute() + } + }) + val response = client.delete("/users/123") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addJwtToken("foo", userId = 123) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } @Test fun `given user identifier equal to current user, 'PATCH users {id} block' must fail`() { - withTestApplication(withRoute { - authenticate { - blockUserRoute() - } - }) { - with(handleRequest(HttpMethod.Patch, "/users/123/block") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + blockUserRoute() + } + }) + val response = client.patch("/users/123/block") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) addJwtToken("foo", userId = 123) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } @Test fun `given user identifier equal to current user, 'PUT users {id}' must fail`() { - withTestApplication(withRoute { - authenticate { - editUserRoute() - } - }) { - with(handleRequest(HttpMethod.Put, "/users/123") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + testApplication { + application(withRoute { + authenticate { + editUserRoute() + } + }) + val response = client.put("/users/123") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = EditUserPayload(role = Role.USER) setBody(Json.encodeToString(EditUserPayload.serializer(), form)) addJwtToken("foo", userId = 123) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } @@ -247,18 +248,15 @@ class UserManagementTest { fun `given external user, 'PATCH users {id} reset-password' must fail`() { wheneverBlocking(userRepository) { findById(123) } .thenReturn( - //null password hash means the user is external UserEntity(id = 123, username = "external-user", passwordHash = null, role = "USER") ) - withTestApplication(withRoute { - resetPasswordRoute() - }) { - with(handleRequest(HttpMethod.Patch, "/users/123/reset-password") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) + testApplication { + application(withRoute { resetPasswordRoute() }) + val response = client.patch("/users/123/reset-password") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } @@ -266,46 +264,45 @@ class UserManagementTest { fun `given external user with external role management, 'PUT users {id}' must fail`() { wheneverBlocking(userRepository) { findById(123) } .thenReturn( - //null password hash means the user is external UserEntity(id = 123, username = "external-user", passwordHash = null, role = "USER") ) - withTestApplication(moduleFunction = { - install(Resources) - install(ContentNegotiation) { - json() - } - di { - bind() with eagerSingleton { userRepository } - bind() with eagerSingleton { passwordService } - bind() with eagerSingleton { passwordGenerator } - bind() with eagerSingleton { - UserManagementServiceImpl( - userRepository = instance(), - passwordService = instance(), - passwordGenerator = instance(), - externalRoleManagement = true - ) + testApplication { + application { + install(Resources) + install(ContentNegotiation) { + json() + } + di { + bind() with eagerSingleton { userRepository } + bind() with eagerSingleton { passwordService } + bind() with eagerSingleton { passwordGenerator } + bind() with eagerSingleton { + UserManagementServiceImpl( + userRepository = instance(), + passwordService = instance(), + passwordGenerator = instance(), + externalRoleManagement = true + ) + } + } + install(StatusPages) { + authStatusPages() + } + routing { + editUserRoute() } } - install(StatusPages) { - authStatusPages() - } - routing { - editUserRoute() - } - }) { - with(handleRequest(HttpMethod.Put, "/users/123") { - addHeader(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + val response = client.put("/users/123") { + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) val form = EditUserPayload(role = Role.ADMIN) setBody(Json.encodeToString(EditUserPayload.serializer(), form)) - }) { - assertEquals(HttpStatusCode.UnprocessableEntity, response.status()) } + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) } } - private fun withRoute(route: Routing.() -> Unit): Application.() -> Unit = { + private fun withRoute(route: Route.() -> Unit): Application.() -> Unit = { install(Resources) install(ContentNegotiation) { json() diff --git a/admin-common/build.gradle.kts b/admin-common/build.gradle.kts index f436471c2..86e4351e8 100644 --- a/admin-common/build.gradle.kts +++ b/admin-common/build.gradle.kts @@ -16,6 +16,7 @@ val ktorVersion: String by parent!!.extra val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val mockitoKotlinVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra val flywaydbVersion: String by parent!!.extra @@ -23,6 +24,7 @@ val testContainersVersion: String by parent!!.extra val postgresSqlVersion: String by parent!!.extra val zaxxerHikaricpVersion: String by parent!!.extra val quartzVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -44,6 +46,7 @@ dependencies { implementation(kotlin("stdlib-jdk8")) implementation("org.jetbrains.kotlinx:kotlinx-datetime:$kotlinxDatetimeVersion") implementation("io.github.microutils:kotlin-logging-jvm:$microutilsLoggingVersion") + implementation("io.micrometer:micrometer-core:$micrometerVersion") implementation("org.kodein.di:kodein-di-framework-ktor-server-jvm:$kodeinVersion") implementation("io.ktor:ktor-server-core:$ktorVersion") implementation("io.ktor:ktor-server-content-negotiation:$ktorVersion") @@ -55,8 +58,6 @@ dependencies { implementation("io.ktor:ktor-client-core:$ktorVersion") implementation("io.ktor:ktor-client-apache:$ktorVersion") - implementation("io.ktor:ktor-client-json:$ktorVersion") - implementation("io.ktor:ktor-client-serialization:$ktorVersion") implementation("io.ktor:ktor-client-content-negotiation:$ktorVersion") implementation("io.ktor:ktor-server-resources:$ktorVersion") implementation("io.ktor:ktor-server-status-pages:$ktorVersion") @@ -66,7 +67,7 @@ dependencies { testImplementation("org.jetbrains.kotlin:kotlin-test:$kotlinVersion") testImplementation("org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion") testImplementation("com.zaxxer:HikariCP:$zaxxerHikaricpVersion") - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") } tasks { diff --git a/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/DatabaseConfig.kt b/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/DatabaseConfig.kt index c2b72c742..5463e36cf 100644 --- a/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/DatabaseConfig.kt +++ b/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/DatabaseConfig.kt @@ -17,6 +17,8 @@ package com.epam.drill.admin.common.config import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers +import io.micrometer.core.instrument.Metrics +import io.micrometer.core.instrument.Timer import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.Transaction import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction @@ -49,6 +51,20 @@ open class DatabaseConfig( .migrate() } - suspend fun transaction(block: suspend Transaction.() -> T): T = - newSuspendedTransaction(dispatcher, database) { block() } + suspend fun transaction(block: suspend Transaction.() -> T): T { + val sample = Timer.start(Metrics.globalRegistry) + return runCatching { + newSuspendedTransaction(dispatcher, database) { block() }.also { + sample.stop(transactionTimer("success")) + } + }.onFailure { + sample.stop(transactionTimer("failure")) + }.getOrThrow() + } + + private fun transactionTimer(status: String) = Metrics.timer( + "drill_db_exposed_transaction_duration", + "schema", dbSchema, + "status", status + ) } \ No newline at end of file diff --git a/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/MeterExtensions.kt b/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/MeterExtensions.kt new file mode 100644 index 000000000..5dbff594b --- /dev/null +++ b/admin-common/src/main/kotlin/com/epam/drill/admin/common/config/MeterExtensions.kt @@ -0,0 +1,71 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.common.config + +import io.micrometer.core.instrument.Timer + +/** + * Measures the duration of a suspended block and records it in this [Timer]. + * + * Unlike [Timer.record], this function correctly handles Kotlin coroutines: + * it captures wall-clock time across suspension points using [Timer.Sample]. + * + * Usage: + * ```kotlin + * val savingLatency = Timer.builder("metrics.savingLatency") + * .description("Latency of saving raw data") + * .register(registry) + * + * savingLatency.recordSuspend { + * someRepository.save(entity) + * } + * ``` + */ +suspend fun Timer.recordSuspend(block: suspend () -> T): T { + val sample = Timer.start() + return try { + block() + } finally { + sample.stop(this) + } +} + +/** + * Measures the duration of a non-suspended block and records it in this [Timer]. + * + * This is a convenient wrapper around [Timer.record] that allows using a lambda + * instead of a [Timer.Sample]. It is suitable for measuring short, non-suspending operations. + * + * Usage: + * ```kotlin + * val processingLatency = Timer.builder("metrics.processingLatency") + * .description("Latency of processing data") + * .register(registry) + * + * processingLatency.recordInline { + * processData(data) + * } + * ``` + */ +inline fun Timer.recordInline(crossinline block: () -> T): T { + val sample = Timer.start() + return try { + block() + } finally { + sample.stop(this) + } +} + diff --git a/admin-etl/build.gradle.kts b/admin-etl/build.gradle.kts index e74591af8..7728c943c 100644 --- a/admin-etl/build.gradle.kts +++ b/admin-etl/build.gradle.kts @@ -17,9 +17,11 @@ val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val flywaydbVersion: String by parent!!.extra val postgresSqlVersion: String by parent!!.extra val zaxxerHikaricpVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -53,6 +55,7 @@ dependencies { implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") + implementation("io.micrometer:micrometer-core:$micrometerVersion") implementation("io.ktor:ktor-server-resources:$ktorVersion") implementation("io.ktor:ktor-server-status-pages:$ktorVersion") @@ -60,7 +63,7 @@ dependencies { compileOnly("org.postgresql:postgresql:${postgresSqlVersion}") testImplementation("org.jetbrains.kotlin:kotlin-test:$kotlinVersion") - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") } tasks { diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/DataTransformer.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/DataTransformer.kt index 0d81fb2e1..5465bfef6 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/DataTransformer.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/DataTransformer.kt @@ -23,14 +23,4 @@ interface DataTransformer { groupId: String, collector: Flow, ): Flow -} - -class NopTransformer : DataTransformer { - override val name: String = "nop-transformer" - override suspend fun transform( - groupId: String, - collector: Flow, - ): Flow = collector -} - -val untypedNopTransformer = NopTransformer() +} \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/EtlPipeline.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/EtlPipeline.kt index c96577245..91f550d62 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/EtlPipeline.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/EtlPipeline.kt @@ -15,37 +15,25 @@ */ package com.epam.drill.admin.etl +import kotlinx.coroutines.flow.Flow import java.time.Instant /** - * EtlPipeline represents a single ETL flow connecting: - * - one `DataExtractor` (source), - * - one or more `DataLoader`-s (sinks). - * It encodes business logic of how raw data becomes metric data. - * - * Core Concepts: - * - **Composition**: EtlPipeline is typically constructed with specific extractor and loaders. - * - **Concurrency control**: may use coroutines to run: - * - extractor in one coroutine, - * - loaders in another, - * - sharing a bounded buffer (`bufferSize`). - * - **Backpressure**: - * - if loaders are slow, the buffer fills up and extractor suspends, keeping memory usage controlled. - * - **Error propagation**: - * - failures in loaders or extractor are propagated to the pipeline, - * - pipeline may cancel child coroutines and report status to `EtlOrchestrator`. + * EtlPipeline represents a linear ETL flow: + * DataExtractor -> DataTransformer -> DataLoader */ -interface EtlPipeline { +interface EtlPipeline { val name: String - val extractor: DataExtractor - val loaders: List, DataLoader>> + val extractor: DataExtractor + val transformer: DataTransformer + val loader: DataLoader suspend fun execute( groupId: String, - sinceTimestampPerLoader: Map, + sinceTimestamp: Instant, untilTimestamp: Instant, - onExtractingProgress: suspend (EtlExtractingResult) -> Unit = {}, - onLoadingProgress: suspend (loaderName: String, result: EtlLoadingResult) -> Unit = { _, _ -> }, - onStatusChanged: suspend (loaderName: String, status: EtlStatus) -> Unit = { _, _ -> }, + extractedFlow: Flow, + onLoadingProgress: suspend (EtlLoadingResult) -> Unit = {}, + onStatusChanged: suspend (EtlStatus) -> Unit = {}, ): EtlProcessingResult suspend fun cleanUp(groupId: String) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlConfig.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlConfig.kt index 5d6ad29fa..347c5aa99 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlConfig.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlConfig.kt @@ -20,7 +20,7 @@ import io.ktor.server.config.ApplicationConfig /** * Configuration parameters for ETL processing. */ -class EtlConfig(private val config: ApplicationConfig) { +class EtlConfig(private val config: ApplicationConfig, internal val metrics: EtlMeter) { /** * Controls the in-memory buffer capacity used for the shared flow between the extractor and loaders. */ diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlMeter.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlMeter.kt new file mode 100644 index 000000000..53f0f3bfd --- /dev/null +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlMeter.kt @@ -0,0 +1,111 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.etl.config + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Timer +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +class EtlMeter(val registry: MeterRegistry) { + + fun rowsFetched(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_fetched", jobName, groupId) + } + + fun rowsExtracted(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_extracted", jobName, groupId) + } + + fun rowsAggregated(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_aggregated", jobName, groupId) + } + + fun aggregationBufferOccupancyRatio(jobName: String, groupId: String): AtomicReference { + return registerDoubleGauge("etl_aggregation_buffer_occupancy_ratio", jobName, groupId) + } + + fun extractionBufferOccupancyRatio(jobName: String, groupId: String): AtomicInteger { + return registerIntegerGauge("etl_extraction_buffer_occupancy_ratio", jobName, groupId) + } + + fun rowsFiltered(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_filtered", jobName, groupId) + } + + fun rowsProcessed(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_processed", jobName, groupId) + } + + fun rowsLoaded(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_loaded", jobName, groupId) + } + + fun rowsSkipped(jobName: String, groupId: String): Counter { + return registerCounter("etl_rows_skipped", jobName, groupId) + } + + fun loadingFailures(jobName: String, groupId: String): Counter { + return registerCounter("etl_loading_failures", jobName, groupId) + } + + fun extractionFailures(jobName: String, groupId: String): Counter { + return registerCounter("etl_extraction_failures", jobName, groupId) + } + + fun loadingDuration(jobName: String, groupId: String): Timer { + return registerTimer("etl_loading_duration", jobName, groupId) + } + + fun extractionDuration(jobName: String, groupId: String): Timer { + return registerTimer("etl_extraction_duration", jobName, groupId) + } + + private fun registerIntegerGauge(metricName: String, jobName: String, groupId: String): AtomicInteger { + val value = AtomicInteger(0) + Gauge.builder(metricName) { value.get() } + .tag("jobName", jobName) + .tag("groupId", groupId) + .register(registry) + return value + } + + private fun registerDoubleGauge(metricName: String, jobName: String, groupId: String): AtomicReference { + val value = AtomicReference(0.0) + Gauge.builder(metricName) { value.get() } + .tag("jobName", jobName) + .tag("groupId", groupId) + .register(registry) + return value + } + + private fun registerCounter(metricName: String, jobName: String, groupId: String): Counter { + return Counter.builder(metricName) + .tag("jobName", jobName) + .tag("groupId", groupId) + .register(registry) + } + + + private fun registerTimer(metricName: String, jobName: String, groupId: String): Timer { + return Timer.builder(metricName) + .tag("jobName", jobName) + .tag("groupId", groupId) + .register(registry) + } +} \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlModule.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlModule.kt index 9322d0955..ca4822a57 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlModule.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/config/EtlModule.kt @@ -21,14 +21,23 @@ import com.epam.drill.admin.etl.EtlOrchestrator import com.epam.drill.admin.etl.impl.EtlMetadataRepositoryImpl import com.epam.drill.admin.etl.impl.EtlOrchestratorImpl import com.epam.drill.admin.etl.job.UpdateMetricsEtlJob -import com.epam.drill.admin.etl.metrics.buildsPipeline -import com.epam.drill.admin.etl.metrics.coveragePipeline -import com.epam.drill.admin.etl.metrics.methodsPipeline -import com.epam.drill.admin.etl.metrics.testDefinitionsPipeline -import com.epam.drill.admin.etl.metrics.testLaunchCoveragePipeline -import com.epam.drill.admin.etl.metrics.testLaunchesPipeline -import com.epam.drill.admin.etl.metrics.testSessionBuildsPipeline -import com.epam.drill.admin.etl.metrics.testSessionsPipeline +import com.epam.drill.admin.etl.pipeline.buildMethodsPipeline +import com.epam.drill.admin.etl.pipeline.buildsPipeline +import com.epam.drill.admin.etl.pipeline.buildMethodCoveragePipeline +import com.epam.drill.admin.etl.pipeline.buildMethodTestSessionCoveragePipeline +import com.epam.drill.admin.etl.pipeline.methodDailyCoveragePipeline +import com.epam.drill.admin.etl.pipeline.testSessionBuildsFromCoveragePipeline +import com.epam.drill.admin.etl.pipeline.methodsPipeline +import com.epam.drill.admin.etl.pipeline.testDefinitionsPipeline +import com.epam.drill.admin.etl.pipeline.buildMethodCoverageFromTestLaunchesPipeline +import com.epam.drill.admin.etl.pipeline.buildMethodTestDefinitionCoveragePipeline +import com.epam.drill.admin.etl.pipeline.buildMethodTestSessionCoverageFromTestLaunchesPipeline +import com.epam.drill.admin.etl.pipeline.methodDailyCoverageFromTestLaunchesPipeline +import com.epam.drill.admin.etl.pipeline.test2CodeMappingPipeline +import com.epam.drill.admin.etl.pipeline.testSessionBuildsFromTestLaunchesPipeline +import com.epam.drill.admin.etl.pipeline.testLaunchesPipeline +import com.epam.drill.admin.etl.pipeline.testSessionBuildsPipeline +import com.epam.drill.admin.etl.pipeline.testSessionsPipeline import com.epam.drill.admin.etl.service.EtlService import com.epam.drill.admin.etl.service.impl.EtlServiceImpl import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig @@ -54,19 +63,40 @@ val etlDIModule ) } bind() with singleton { + val metrics = EtlMeter(instance()) val drillConfig: ApplicationConfig = instance().environment.config.config("drill") - val etlConfig = EtlConfig(drillConfig.config("etl")) + val etlConfig = EtlConfig(drillConfig.config("etl"), metrics) + with(etlConfig) { EtlOrchestratorImpl( name = "metrics", pipelines = listOf( - buildsPipeline, methodsPipeline, - testLaunchesPipeline, testDefinitionsPipeline, testSessionsPipeline, - coveragePipeline, testLaunchCoveragePipeline, testSessionBuildsPipeline + // Reference data + buildsPipeline, + // build_methods extractor group + buildMethodsPipeline, + methodsPipeline, + testLaunchesPipeline, + testDefinitionsPipeline, + testSessionsPipeline, + testSessionBuildsPipeline, + // Coverage extractor group + buildMethodTestSessionCoveragePipeline, + buildMethodCoveragePipeline, + methodDailyCoveragePipeline, + testSessionBuildsFromCoveragePipeline, + // Test-launch coverage extractor group + buildMethodTestDefinitionCoveragePipeline, + buildMethodTestSessionCoverageFromTestLaunchesPipeline, + buildMethodCoverageFromTestLaunchesPipeline, + methodDailyCoverageFromTestLaunchesPipeline, + test2CodeMappingPipeline, + testSessionBuildsFromTestLaunchesPipeline, ), metadataRepository = instance(), - consistencyWindow = etlConfig.consistencyWindow, - processingDelay = etlConfig.processingDelay + consistencyWindow = consistencyWindow, + processingDelay = processingDelay, + bufferSize = bufferSize, ) } } diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/CompletableSharedFlow.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/CompletableSharedFlow.kt index cbb461237..674c4d0f3 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/CompletableSharedFlow.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/CompletableSharedFlow.kt @@ -23,6 +23,7 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.mapNotNull import kotlinx.coroutines.flow.takeWhile import kotlinx.coroutines.withTimeoutOrNull +import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.seconds sealed interface Event { diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/LruMap.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/LruMap.kt index 1e7cfc7f6..598be5acd 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/LruMap.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/flow/LruMap.kt @@ -20,31 +20,34 @@ package com.epam.drill.admin.etl.flow * when the maximum size is reached. */ class LruMap( - private val maxSize: Int, - private val onEvict: (K, V) -> Unit + private val maxSize: Int ) { private val map = LinkedHashMap(16, 0.75f, true) val size: Int get() = map.size - fun compute(key: K, update: (V?) -> V) { + fun compute(key: K, update: (V?) -> V): V? { map[key] = update(map[key]) - if (map.size >= maxSize) { + return if (map.size > maxSize) { evictOldest() + } else { + null } } - fun evictOldest() { + fun evictOldest(): V? { val it = map.entries.iterator() - if (!it.hasNext()) return + if (!it.hasNext()) return null val entry = it.next() it.remove() - onEvict(entry.key, entry.value) + return entry.value } - fun evictAll() { + suspend fun evictAll( + onEvict: suspend (key: K, value: V) -> Unit + ) { val it = map.entries.iterator() while (it.hasNext()) { val entry = it.next() diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoader.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoader.kt index 3ee84ab87..fbe4de818 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoader.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoader.kt @@ -21,17 +21,18 @@ import com.epam.drill.admin.etl.EtlRow import com.epam.drill.admin.etl.EtlStatus import com.epam.drill.admin.etl.flow.StoppableFlow import com.epam.drill.admin.etl.flow.stoppable +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.flow.Flow import mu.KotlinLogging import java.time.Instant import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration.Companion.seconds abstract class BatchDataLoader( override val name: String, open val batchSize: Int = 1000, open val loggingFrequency: Int = 10, + open val metrics: EtlMeter ) : DataLoader { private val logger = KotlinLogging.logger {} @@ -53,10 +54,11 @@ abstract class BatchDataLoader( var result = EtlLoadingResult(lastProcessedAt = sinceTimestamp) val flow = collector.stoppable() val batchNo = AtomicInteger(0) - val loadedRows = AtomicLong(0) - val skippedRows = AtomicLong(0) + val processedRows = metrics.rowsProcessed(name, groupId) + val loadedRows = metrics.rowsLoaded(name, groupId) + val skippedRows = metrics.rowsSkipped(name, groupId) + var isLoadingStarted = false val buffer = mutableListOf() - var skippedRowsForUpdate = 0L var lastLoadedTimestamp: Instant = sinceTimestamp var previousTimestamp: Instant? = null suspend fun StoppableFlow.stopWithMessage(message: String) { @@ -71,10 +73,12 @@ abstract class BatchDataLoader( trackProgressOf { flow.collect { row -> - if (loadedRows.get() == 0L && skippedRows.get() == 0L) { + if (!isLoadingStarted) { logger.debug { "ETL loader [$name] for group [$groupId] loading rows..." } onStatusChanged(EtlStatus.LOADING) + isLoadingStarted = true } + processedRows.increment() val currentTimestamp = row.timestamp if (previousTimestamp != null && currentTimestamp < previousTimestamp) { flow.stopWithMessage("Timestamps in the extracted data are not in ascending order: $currentTimestamp < $previousTimestamp") @@ -84,7 +88,7 @@ abstract class BatchDataLoader( // Skip rows that are already processed if (currentTimestamp <= sinceTimestamp) { previousTimestamp = currentTimestamp - skippedRows.incrementAndGet() + skippedRows.increment() return@collect } @@ -99,6 +103,7 @@ abstract class BatchDataLoader( if (batch.success) { lastLoadedTimestamp = previousTimestamp ?: throw IllegalStateException("Previous timestamp is null") + loadedRows.increment(batch.rowsLoaded.toDouble()) } EtlLoadingResult( errorMessage = if (!batch.success) result.errorMessage else null, @@ -116,35 +121,15 @@ abstract class BatchDataLoader( return@collect } - // Skip rows that are not processable - if (!isProcessable(row)) { - previousTimestamp = currentTimestamp - skippedRows.incrementAndGet() - skippedRowsForUpdate++ - // If timestamp changed and there are a lot of skipped rows, update progress - if (previousTimestamp != null && currentTimestamp != previousTimestamp && buffer.isEmpty() && skippedRowsForUpdate >= batchSize) { - onLoadingProgress( - EtlLoadingResult( - lastProcessedAt = previousTimestamp - ?: throw IllegalStateException("Previous timestamp is null"), - processedRows = 0, - ) - ) - skippedRowsForUpdate = 0 - } - return@collect - } - buffer += row previousTimestamp = currentTimestamp - loadedRows.incrementAndGet() } }.every(loggingFrequency.seconds) { - if (loadedRows.get() > 0L || skippedRows.get() > 0L) { + if (isLoadingStarted) { logger.debug { - "ETL loader [$name] for group [$groupId] loaded ${loadedRows.get()} rows" + + "ETL loader [$name] for group [$groupId] loaded ${loadedRows.count().toLong()} rows" + ", batch: ${batchNo.get()}" + - ", skipped rows: ${skippedRows.get()}" + ", skipped rows: ${skippedRows.count().toLong()}" } } @@ -157,6 +142,7 @@ abstract class BatchDataLoader( if (batch.success) { lastLoadedTimestamp = previousTimestamp ?: throw IllegalStateException("Previous timestamp is null") + loadedRows.increment(batch.rowsLoaded.toDouble()) } EtlLoadingResult( errorMessage = if (!batch.success) batch.errorMessage else null, @@ -186,7 +172,6 @@ abstract class BatchDataLoader( return result } - abstract fun isProcessable(args: T): Boolean abstract suspend fun loadBatch(groupId: String, batch: List, batchNo: Int): BatchResult private suspend fun flushBuffer( diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlOrchestratorImpl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlOrchestratorImpl.kt index c6b865fae..66945dc3a 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlOrchestratorImpl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlOrchestratorImpl.kt @@ -1,4 +1,4 @@ -/** +/** * Copyright 2020 - 2022 EPAM Systems * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,7 @@ */ package com.epam.drill.admin.etl.impl +import com.epam.drill.admin.etl.DataExtractor import com.epam.drill.admin.etl.EtlExtractingResult import com.epam.drill.admin.etl.EtlLoadingResult import com.epam.drill.admin.etl.EtlMetadata @@ -24,14 +25,12 @@ import com.epam.drill.admin.etl.EtlPipeline import com.epam.drill.admin.etl.EtlProcessingResult import com.epam.drill.admin.etl.EtlRow import com.epam.drill.admin.etl.EtlStatus -import io.ktor.util.pipeline.Pipeline +import com.epam.drill.admin.etl.flow.CompletableSharedFlow import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.withContext import mu.KotlinLogging import java.time.Instant @@ -45,6 +44,7 @@ open class EtlOrchestratorImpl( open val metadataRepository: EtlMetadataRepository, open val consistencyWindow: Long = 0, open val processingDelay: Long = 0, + open val bufferSize: Int = 2000, ) : EtlOrchestrator { private val logger = KotlinLogging.logger {} @@ -54,9 +54,18 @@ open class EtlOrchestratorImpl( val results = Collections.synchronizedList(mutableListOf()) val duration = measureTimeMillis { trackProgressOf { - pipelines.map { pipeline -> + // Group pipelines by extractor name; pipelines in the same group share one extractor run + val extractorGroups = pipelines.groupBy { it.extractor.name } + extractorGroups.map { (_, groupedPipelines) -> async { - results += runPipeline(groupId, pipeline, initTimestamp) + @Suppress("UNCHECKED_CAST") + val typedPipelines = groupedPipelines as List> + results += runPipelineGroupByExtractor( + groupId = groupId, + groupedPipelines = typedPipelines, + extractor = typedPipelines.first().extractor, + initTimestamp = initTimestamp + ) } }.awaitAll() }.every(1.minutes) { @@ -77,7 +86,7 @@ open class EtlOrchestratorImpl( override suspend fun rerun( groupId: String, initTimestamp: Instant, - withDataDeletion: Boolean + withDataDeletion: Boolean, ): List = withContext(Dispatchers.IO) { logger.info { "ETL [$name] for group [$groupId] is deleting all metadata for rerun..." } @@ -90,120 +99,170 @@ open class EtlOrchestratorImpl( pipelines.forEach { it.cleanUp(groupId) } logger.info { "ETL [$name] for group [$groupId] deleted all data for rerun." } } - val results = run(groupId, initTimestamp) - return@withContext results + return@withContext run(groupId, initTimestamp) } - private suspend fun runPipeline( + /** + * Runs a group of pipelines that share the same extractor. + * The extractor is executed exactly once; its output is broadcast to all pipelines in the group. + */ + private suspend fun runPipelineGroupByExtractor( groupId: String, - pipeline: EtlPipeline<*, *>, - initTimestamp: Instant - ): EtlProcessingResult = coroutineScope { + groupedPipelines: List>, + extractor: DataExtractor = groupedPipelines.first().extractor, + initTimestamp: Instant, + ): List = coroutineScope { val snapshotTime = Instant.now().minusSeconds(processingDelay) - val metadata = metadataRepository.getAllMetadataByExtractor(groupId, pipeline.name, pipeline.extractor.name) - .associateBy { it.loaderName } - val loaderNames = pipeline.loaders.map { it.second.name }.toSet() - val timestampPerLoader = loaderNames.associateWith { - val lastProcessedAt = metadata[it]?.lastProcessedAt - if (lastProcessedAt != null) { - lastProcessedAt.minusSeconds(consistencyWindow) - } else { + + // Compute per-pipeline sinceTimestamp from metadata + val sinceTimestamps: Map = groupedPipelines.associate { pipeline -> + val metadata = metadataRepository.getMetadata( + groupId, pipeline.name, extractor.name, pipeline.loader.name + ) + val sinceTimestamp = if (metadata?.lastProcessedAt != null) + metadata.lastProcessedAt.minusSeconds(consistencyWindow) + else initTimestamp - } + pipeline.name to sinceTimestamp } - try { - for (loader in loaderNames) { + for (pipeline in groupedPipelines) { + try { metadataRepository.saveMetadata( EtlMetadata( groupId = groupId, pipelineName = pipeline.name, - extractorName = pipeline.extractor.name, - loaderName = loader, + extractorName = extractor.name, + loaderName = pipeline.loader.name, status = EtlStatus.EXTRACTING, - lastProcessedAt = timestampPerLoader[loader] ?: initTimestamp, + lastProcessedAt = sinceTimestamps[pipeline.name] ?: initTimestamp, lastRunAt = snapshotTime, ) ) + } catch (e: Throwable) { + logger.warn( + "ETL pipeline [${pipeline.name}] for group [$groupId] failed to save initial metadata: ${e.message}", + e + ) + } + } + + val minLastProcessedAt = sinceTimestamps.values.min() + val sharedFlow = CompletableSharedFlow(replay = 0, extraBufferCapacity = bufferSize) + val jobs = groupedPipelines.map { pipeline -> + async { + runPipelineWithExtractionFlow( + groupId = groupId, + pipeline = pipeline, + sinceTimestamp = sinceTimestamps[pipeline.name] ?: initTimestamp, + untilTimestamp = snapshotTime, + sharedFlow = sharedFlow, + ) } - val pipelineResult = pipeline.execute( + } + sharedFlow.waitForSubscribers(jobs.count { it.isActive }) + try { + extractor.extract( groupId = groupId, - sinceTimestampPerLoader = timestampPerLoader, + sinceTimestamp = minLastProcessedAt, untilTimestamp = snapshotTime, + emitter = sharedFlow, onExtractingProgress = { result -> - progressExtracting( - groupId = groupId, - pipelineName = pipeline.name, - extractorName = pipeline.extractor.name, - result = result - ) - }, - onLoadingProgress = { loaderName, result -> - progressLoading( - groupId = groupId, - pipelineName = pipeline.name, - extractorName = pipeline.extractor.name, - loaderName = loaderName, - result = result - ) - }, - onStatusChanged = { loaderName, status -> - try { - metadataRepository.accumulateMetadataByLoader( - groupId = groupId, - pipelineName = pipeline.name, - extractorName = pipeline.extractor.name, - loaderName = loaderName, - status = status - ) - } catch (e: Throwable) { - logger.warn( - "ETL pipeline [${pipeline.name}] for group [$groupId] failed to update loading status: ${e.message}", - e - ) + groupedPipelines.forEach { pipeline -> + progressExtracting(groupId, pipeline.name, extractor.name, result) } } ) - return@coroutineScope pipelineResult } catch (e: Throwable) { - logger.error("ETL pipeline [${pipeline.name}] for group [$groupId] failed: ${e.message}", e) - return@coroutineScope EtlProcessingResult( - groupId = groupId, - pipelineName = pipeline.name, - lastProcessedAt = initTimestamp, - rowsProcessed = 0, - status = EtlStatus.FAILED, - errorMessage = e.message + logger.debug(e) { "ETL extractor [${extractor.name}] for group [$groupId] failed: ${e.message}" } + val errorResult = EtlExtractingResult( + errorMessage = "Error during extracting data with extractor ${extractor.name}: ${e.message ?: e.javaClass.simpleName}" ) + groupedPipelines.forEach { pipeline -> + progressExtracting(groupId, pipeline.name, extractor.name, errorResult) + } + } finally { + sharedFlow.complete() } + + jobs.awaitAll() + } + + /** + * Runs a single pipeline with the provided shared flow of extracted data. + */ + private suspend fun runPipelineWithExtractionFlow( + groupId: String, + pipeline: EtlPipeline, + sinceTimestamp: Instant, + untilTimestamp: Instant, + sharedFlow: Flow, + ): EtlProcessingResult = try { + pipeline.execute( + groupId = groupId, + sinceTimestamp = sinceTimestamp, + untilTimestamp = untilTimestamp, + extractedFlow = sharedFlow, + onLoadingProgress = { result -> + progressLoading(groupId, pipeline.name, pipeline.extractor.name, pipeline.loader.name, result) + }, + onStatusChanged = { status -> + try { + metadataRepository.accumulateMetadataByLoader( + groupId = groupId, + pipelineName = pipeline.name, + extractorName = pipeline.extractor.name, + loaderName = pipeline.loader.name, + status = status, + ) + } catch (e: Throwable) { + logger.warn( + "ETL pipeline [${pipeline.name}] for group [$groupId] failed to update loading status: ${e.message}", + e + ) + } + } + ) + } catch (e: Throwable) { + logger.error("ETL pipeline [${pipeline.name}] for group [$groupId] failed: ${e.message}", e) + EtlProcessingResult( + groupId = groupId, + pipelineName = pipeline.name, + lastProcessedAt = sinceTimestamp, + rowsProcessed = 0, + status = EtlStatus.FAILED, + errorMessage = e.message, + ) } - suspend fun progressExtracting(groupId: String, - pipelineName: String, - extractorName: String, - result: EtlExtractingResult) { + private suspend fun progressExtracting( + groupId: String, + pipelineName: String, + extractorName: String, + result: EtlExtractingResult, + ) { try { metadataRepository.accumulateMetadataByExtractor( groupId = groupId, pipelineName = pipelineName, extractorName = extractorName, errorMessage = result.errorMessage, - extractDuration = result.duration + extractDuration = result.duration, ) } catch (e: Throwable) { logger.warn( - "ETL pipeline [${pipelineName}] for group [$groupId] failed to update extracting progress: ${e.message}", + "ETL pipeline [$pipelineName] for group [$groupId] failed to update extracting progress: ${e.message}", e ) } } - suspend fun progressLoading( + private suspend fun progressLoading( groupId: String, pipelineName: String, extractorName: String, loaderName: String, - result: EtlLoadingResult + result: EtlLoadingResult, ) { try { metadataRepository.accumulateMetadataByLoader( @@ -214,12 +273,11 @@ open class EtlOrchestratorImpl( errorMessage = result.errorMessage, lastProcessedAt = result.lastProcessedAt, loadDuration = result.duration ?: 0L, - rowsProcessed = result.processedRows + rowsProcessed = result.processedRows, ) } catch (e: Throwable) { logger.warn( - "ETL pipeline [$pipelineName] for group [$groupId] failed to update loading progress: ${e.message}", - e + "ETL pipeline [$pipelineName] for group [$groupId] failed to update loading progress: ${e.message}", e ) } } diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlPipelineImpl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlPipelineImpl.kt index 0a4e2df31..4366ff38a 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlPipelineImpl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/EtlPipelineImpl.kt @@ -1,4 +1,4 @@ -/** +/** * Copyright 2020 - 2022 EPAM Systems * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,20 +18,15 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.DataExtractor import com.epam.drill.admin.etl.DataLoader import com.epam.drill.admin.etl.DataTransformer -import com.epam.drill.admin.etl.EtlExtractingResult import com.epam.drill.admin.etl.EtlPipeline import com.epam.drill.admin.etl.EtlProcessingResult import com.epam.drill.admin.etl.EtlLoadingResult import com.epam.drill.admin.etl.EtlRow import com.epam.drill.admin.etl.EtlStatus -import com.epam.drill.admin.etl.NopTransformer -import com.epam.drill.admin.etl.flow.CompletableSharedFlow -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.withContext import mu.KotlinLogging import java.time.Instant @@ -40,165 +35,61 @@ import kotlin.system.measureTimeMillis class EtlPipelineImpl( override val name: String, override val extractor: DataExtractor, - override val loaders: List, DataLoader>>, - private val bufferSize: Int = 2000 + override val transformer: DataTransformer, + override val loader: DataLoader, + private val metrics: EtlMeter, ) : EtlPipeline { private val logger = KotlinLogging.logger {} - companion object { - fun singleLoader( - name: String, - extractor: DataExtractor, - loader: DataLoader, - bufferSize: Int = 2000 - ) = EtlPipelineImpl( - name = name, - extractor = extractor, - loaders = listOf(NopTransformer() to loader), - bufferSize = bufferSize - ) - - fun singleLoader( - name: String, - extractor: DataExtractor, - transformer: DataTransformer, - loader: DataLoader, - bufferSize: Int = 2000 - ) = EtlPipelineImpl( - name = name, - extractor = extractor, - loaders = listOf(transformer to loader), - bufferSize = bufferSize - ) - - fun multiLoaders( - name: String, - extractor: DataExtractor, - loaders: List>, - bufferSize: Int = 2000 - ) = EtlPipelineImpl( - name = name, - extractor = extractor, - loaders = loaders.map { NopTransformer() to it }, - bufferSize = bufferSize - ) - } - override suspend fun execute( groupId: String, - sinceTimestampPerLoader: Map, + sinceTimestamp: Instant, untilTimestamp: Instant, - onExtractingProgress: suspend (EtlExtractingResult) -> Unit, - onLoadingProgress: suspend (String, EtlLoadingResult) -> Unit, - onStatusChanged: suspend (loaderName: String, status: EtlStatus) -> Unit + extractedFlow: Flow, + onLoadingProgress: suspend (EtlLoadingResult) -> Unit, + onStatusChanged: suspend (EtlStatus) -> Unit, ): EtlProcessingResult = withContext(Dispatchers.IO) { - val minProcessedTime = sinceTimestampPerLoader.values.min() - logger.debug { "ETL pipeline [$name] for group [$groupId] starting since $minProcessedTime..." } - var results = EtlLoadingResult(lastProcessedAt = minProcessedTime) + logger.debug { "ETL pipeline [$name] for group [$groupId] loading since $sinceTimestamp..." } + var result = EtlLoadingResult(lastProcessedAt = sinceTimestamp) val duration = measureTimeMillis { - results = processEtl( - groupId, - minProcessedTime, - sinceTimestampPerLoader, - untilTimestamp, - onExtractingProgress, - onLoadingProgress, - onStatusChanged - ) + result = loadData(groupId, sinceTimestamp, untilTimestamp, extractedFlow, onLoadingProgress, onStatusChanged) } logger.debug { - if (results.processedRows == 0L && !results.isFailed) { + if (result.processedRows == 0L && !result.isFailed) { "ETL pipeline [$name] for group [$groupId] completed in ${duration}ms, no new rows" } else { - val errors = results.errorMessage?.let { ", errors: $it" } ?: "" - "ETL pipeline [$name] for group [$groupId] completed in ${duration}ms, rows processed: ${results.processedRows}" + errors + val errors = result.errorMessage?.let { ", errors: $it" } ?: "" + "ETL pipeline [$name] for group [$groupId] completed in ${duration}ms, rows processed: ${result.processedRows}" + errors } } EtlProcessingResult( groupId = groupId, pipelineName = name, - lastProcessedAt = results.lastProcessedAt, - rowsProcessed = results.processedRows, - errorMessage = results.errorMessage, - status = if (results.isFailed) EtlStatus.FAILED else EtlStatus.SUCCESS + lastProcessedAt = result.lastProcessedAt, + rowsProcessed = result.processedRows, + errorMessage = result.errorMessage, + status = if (result.isFailed) EtlStatus.FAILED else EtlStatus.SUCCESS ) } override suspend fun cleanUp(groupId: String) { - loaders.forEach { it.second.deleteAll(groupId) } - } - - private suspend fun CoroutineScope.processEtl( - groupId: String, - extractorSinceTimestamp: Instant, - sinceTimestampPerLoader: Map, - untilTimestamp: Instant, - onExtractingProgress: suspend (EtlExtractingResult) -> Unit, - onLoadingProgress: suspend (loaderName: String, EtlLoadingResult) -> Unit, - onStatusChanged: suspend (loaderName: String, status: EtlStatus) -> Unit - ): EtlLoadingResult { - val flow = CompletableSharedFlow( - replay = 0, - extraBufferCapacity = bufferSize - ) - return loaders.map { loader -> - async { - loadData( - groupId, - loader.first, - loader.second, - sinceTimestampPerLoader[loader.second.name] ?: extractorSinceTimestamp, - untilTimestamp, - flow, - onLoadingProgress, - onStatusChanged - ) - } - }.also { jobs -> - extractData(groupId, flow, jobs, extractorSinceTimestamp, untilTimestamp, onExtractingProgress) - }.awaitAll().max() - } - - private suspend fun extractData( - groupId: String, - flow: CompletableSharedFlow, - jobs: List>, - sinceTimestamp: Instant, - untilTimestamp: Instant, - onExtractingProgress: suspend (EtlExtractingResult) -> Unit, - ) { - try { - // Start extractor only after all jobs are ready to consume data otherwise data may be lost - flow.waitForSubscribers(jobs.count { it.isActive }) - extractor.extract(groupId, sinceTimestamp, untilTimestamp, flow, onExtractingProgress) - } catch (e: Throwable) { - logger.debug(e) { "ETL pipeline [$name] for group [$groupId] failed for extractor [${extractor.name}]: ${e.message}" } - onExtractingProgress( - EtlExtractingResult( - errorMessage = "Error during extracting data with extractor ${extractor.name}: ${e.message ?: e.javaClass.simpleName}", - ) - ) - } finally { - // Complete the flow to signal jobs that extraction is done - flow.complete() - } + loader.deleteAll(groupId) } private suspend fun loadData( groupId: String, - transformer: DataTransformer, - loader: DataLoader, sinceTimestamp: Instant, untilTimestamp: Instant, - flow: Flow, - onLoadingProgress: suspend (loaderName: String, result: EtlLoadingResult) -> Unit, - onStatusChanged: suspend (loaderName: String, status: EtlStatus) -> Unit + extractedFlow: Flow, + onLoadingProgress: suspend (EtlLoadingResult) -> Unit, + onStatusChanged: suspend (EtlStatus) -> Unit, ): EtlLoadingResult = try { - transformer.transform(groupId, flow).let { flow -> + val metered = getExtractionFlow(groupId, extractedFlow) + transformer.transform(groupId, metered).let { transformed -> loader.load( - groupId, sinceTimestamp, untilTimestamp, flow, - onLoadingProgress = { onLoadingProgress(loader.name, it) }, - onStatusChanged = { onStatusChanged(loader.name, it) } + groupId, sinceTimestamp, untilTimestamp, transformed, + onLoadingProgress = onLoadingProgress, + onStatusChanged = onStatusChanged, ) } } catch (e: Throwable) { @@ -206,6 +97,16 @@ class EtlPipelineImpl( EtlLoadingResult( errorMessage = "Error during loading data with loader ${loader.name}: ${e.message ?: e.javaClass.simpleName}", lastProcessedAt = sinceTimestamp - ).also { onLoadingProgress(loader.name, it) } + ).also { onLoadingProgress(it) } + } + + private fun getExtractionFlow(groupId: String, flow: Flow): Flow { + val rowsExtracted = metrics.rowsExtracted(name, groupId) + return flow { + flow.collect { row -> + rowsExtracted.increment() + emit(row) + } + } } } \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractor.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractor.kt index 2d7c7311b..3c24b1422 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractor.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractor.kt @@ -18,18 +18,19 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.DataExtractor import com.epam.drill.admin.etl.EtlExtractingResult import com.epam.drill.admin.etl.EtlRow +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.flow.FlowCollector import mu.KotlinLogging import java.time.Instant import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration.Companion.seconds abstract class PageDataExtractor( override val name: String, open val extractionLimit: Int, private val loggingFrequency: Int = 10, + open val metrics: EtlMeter ) : DataExtractor { private val logger = KotlinLogging.logger {} @@ -42,7 +43,8 @@ abstract class PageDataExtractor( ) { var currentSince = sinceTimestamp val page = AtomicInteger(0) - val rowsFetched = AtomicLong(0) + val rowsFetched = metrics.rowsFetched(name, groupId) + val failures = metrics.extractionFailures(name, groupId) var hasMore = true val buffer: MutableList = mutableListOf() val isExecutingQuery = AtomicBoolean(true) @@ -89,7 +91,7 @@ abstract class PageDataExtractor( buffer.add(row) previousTimestamp = currentTimestamp - rowsFetched.incrementAndGet() + rowsFetched.increment() } ) if (pageRows == 0L || pageRows < extractionLimit) { @@ -98,7 +100,7 @@ abstract class PageDataExtractor( previousEmittedTimestamp = previousTimestamp logger.debug { "ETL extractor [$name] for group [$groupId] completed fetching" + - ", rows fetched: ${rowsFetched.get()}" + + ", rows fetched: ${rowsFetched.count()}" + ", total pages: ${page.get()}" + (if (previousEmittedTimestamp != null) ", last extracted at $previousEmittedTimestamp" else "") } @@ -117,6 +119,7 @@ abstract class PageDataExtractor( logger.error { "Error during data extraction with extractor [$name]: ${e.message ?: e.javaClass.simpleName}" } + failures.increment() onExtractingProgress( EtlExtractingResult( errorMessage = e.message @@ -130,7 +133,7 @@ abstract class PageDataExtractor( } } else { logger.debug { - "ETL extractor [$name] for group [$groupId] fetched ${rowsFetched.get()} rows" + + "ETL extractor [$name] for group [$groupId] fetched ${rowsFetched.count()} rows" + ", page: ${page.get()}" } } diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilder.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilder.kt new file mode 100644 index 000000000..61f1bc27f --- /dev/null +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilder.kt @@ -0,0 +1,193 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.etl.impl + +import com.epam.drill.admin.etl.DataExtractor +import com.epam.drill.admin.etl.DataLoader +import com.epam.drill.admin.etl.DataTransformer +import com.epam.drill.admin.etl.EtlRow +import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlConfig +import kotlinx.coroutines.flow.Flow + +/** + * Entry point for the fluent ETL pipeline builder DSL. + */ +fun EtlConfig.pipeline(name: String): PipelineNameStep = + PipelineNameStep(name = name, etlConfig = this) + +class PipelineNameStep internal constructor( + val name: String, + internal val etlConfig: EtlConfig, +) { + fun extractWith(extractor: DataExtractor): ExtractStep = + ExtractStep(name = name, etlConfig = etlConfig, extractor = extractor) +} + +class ExtractStep internal constructor( + val name: String, + internal val etlConfig: EtlConfig, + val extractor: DataExtractor, +) { + /** + * Add a filter step that excludes rows for which [predicate] returns false. + */ + fun filter( + predicate: (UntypedRow) -> Boolean, + ): TransformStep = TransformStep( + name = name, + etlConfig = etlConfig, + extractor = extractor, + transformer = UntypedFilterTransformer( + name = name, + metrics = etlConfig.metrics, + predicate = predicate, + ) + ) + + /** + * Add an aggregation step that groups rows by [groupKeys]. + */ + fun aggregateBy( + vararg groupKeys: String, + aggregate: (current: UntypedRow, next: UntypedRow) -> UntypedRow, + ): TransformStep = TransformStep( + name = name, + etlConfig = etlConfig, + extractor = extractor, + transformer = UntypedAggregationTransformer( + name = name, + bufferSize = etlConfig.transformationBufferSize, + loggingFrequency = etlConfig.loggingFrequency, + metrics = etlConfig.metrics, + groupKeys = groupKeys.toList(), + aggregate = aggregate, + ) + ) + + /** + * Add a custom transformation step that applies the provided [transformer] to the extracted rows. + */ + fun transformWith( + transformer: DataTransformer, + ): TransformStep = TransformStep( + name = name, + etlConfig = etlConfig, + extractor = extractor, + transformer = transformer, + ) + + /** + * Attach a loader — terminates the pipeline with no transformation. + */ + fun loadWith( + loader: DataLoader, + ): EtlPipelineImpl = + EtlPipelineImpl( + name = name, + extractor = extractor, + transformer = NoOpTransformer, + loader = loader, + metrics = etlConfig.metrics, + ) +} + +class TransformStep internal constructor( + val name: String, + internal val etlConfig: EtlConfig, + val extractor: DataExtractor, + val transformer: DataTransformer, +) { + fun filter( + predicate: (UntypedRow) -> Boolean, + ): TransformStep = append( + UntypedFilterTransformer( + name = name, + metrics = etlConfig.metrics, + predicate = predicate, + ) + ) + + /** + * Chain an additional aggregation step. + */ + fun aggregateBy( + vararg groupKeys: String, + aggregate: (current: UntypedRow, next: UntypedRow) -> UntypedRow, + ): TransformStep = append( + UntypedAggregationTransformer( + name = name, + bufferSize = etlConfig.transformationBufferSize, + loggingFrequency = etlConfig.loggingFrequency, + metrics = etlConfig.metrics, + groupKeys = groupKeys.toList(), + aggregate = aggregate, + ) + ) + + /** + * Chain an additional custom transformation step. + */ + fun transformWith( + transformer: DataTransformer, + ): TransformStep = append(transformer) + + /** + * Attach a loader to the end of the transformation chain. + */ + fun loadWith( + loader: DataLoader, + ): EtlPipelineImpl = + EtlPipelineImpl( + name = name, + extractor = extractor, + transformer = transformer, + loader = loader, + metrics = etlConfig.metrics, + ) + + private fun append(next: DataTransformer): TransformStep = + TransformStep( + name = name, + etlConfig = etlConfig, + extractor = extractor, + transformer = SequencedTransformer(transformer, next), + ) +} + +/** + * Composes two [DataTransformer]s in sequence, so that the output of the first one is fed into the second one. + */ +internal class SequencedTransformer( + private val first: DataTransformer, + private val second: DataTransformer +) : DataTransformer { + override val name: String = "${first.name}+${second.name}" + override suspend fun transform(groupId: String, collector: Flow): Flow = + second.transform(groupId, first.transform(groupId, collector)) +} + +/** + * No-op transformer used when [ExtractStep.loadWith] is called without any prior transform step. + */ +internal object NoOpTransformer : DataTransformer { + override val name: String = "identity" + override suspend fun transform( + groupId: String, + collector: Flow, + ): Flow = collector +} + diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataExtractor.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataExtractor.kt index 25ee09e5e..8bc64fe10 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataExtractor.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataExtractor.kt @@ -15,8 +15,10 @@ */ package com.epam.drill.admin.etl.impl +import com.epam.drill.admin.common.config.recordInline import com.epam.drill.admin.etl.EtlRow import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.Dispatchers import mu.KotlinLogging import org.jetbrains.exposed.sql.Database @@ -26,6 +28,8 @@ import java.sql.ResultSet import java.sql.ResultSetMetaData import java.time.Instant import kotlin.system.measureTimeMillis +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.toJavaDuration import kotlin.use abstract class SqlDataExtractor( @@ -35,7 +39,8 @@ abstract class SqlDataExtractor( open val database: Database, open val fetchSize: Int, open val loggingFrequency: Int, -) : PageDataExtractor(name, extractionLimit, loggingFrequency) { + override val metrics: EtlMeter +) : PageDataExtractor(name, extractionLimit, loggingFrequency, metrics) { private val logger = KotlinLogging.logger {} override suspend fun extractPage( @@ -46,24 +51,26 @@ abstract class SqlDataExtractor( onExtractionExecuted: suspend (Long) -> Unit, rowsExtractor: suspend (T) -> Unit ) { + val timer = metrics.extractionDuration(name, groupId) val preparedSql = prepareSql(sqlQuery) - execSuspend( sql = preparedSql.getSql(), args = preparedSql.getArgs( - UntypedRow(sinceTimestamp, mapOf( - "group_id" to groupId, - "since_timestamp" to java.sql.Timestamp.from(sinceTimestamp), - "until_timestamp" to java.sql.Timestamp.from(untilTimestamp), - "limit" to limit, - )) - ), + UntypedRow( + sinceTimestamp, mapOf( + "group_id" to groupId, + "since_timestamp" to java.sql.Timestamp.from(sinceTimestamp), + "until_timestamp" to java.sql.Timestamp.from(untilTimestamp), + "limit" to limit, + ) + ) + ) ) { rs, duration -> + timer.record(duration.milliseconds.toJavaDuration()) + onExtractionExecuted(duration) val meta = rs.metaData val columnCount = meta.columnCount - - onExtractionExecuted(duration) - while (rs.next()) { + while (timer.recordInline { rs.next() }) { val row = parseRow(rs, meta, columnCount) rowsExtractor(row) } @@ -90,14 +97,11 @@ abstract class SqlDataExtractor( } else stmt.setNull(index + 1, TextColumnType()) } - - val resultSet: ResultSet + val rs: ResultSet val duration = measureTimeMillis { - resultSet = stmt.executeQuery() - } - resultSet.use { rs -> - collect(rs, duration) + rs = stmt.executeQuery() } + collect(rs, duration) } finally { stmt.closeIfPossible() } diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataLoader.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataLoader.kt index a4ae9ace6..c26ed4297 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataLoader.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataLoader.kt @@ -15,7 +15,9 @@ */ package com.epam.drill.admin.etl.impl +import com.epam.drill.admin.common.config.recordInline import com.epam.drill.admin.etl.EtlRow +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.Dispatchers import mu.KotlinLogging import org.jetbrains.exposed.sql.Database @@ -33,8 +35,9 @@ abstract class SqlDataLoader( override val loggingFrequency: Int, open val sqlUpsert: String, open val sqlDelete: String, - open val database: Database -) : BatchDataLoader(name, batchSize, loggingFrequency) { + open val database: Database, + override val metrics: EtlMeter +) : BatchDataLoader(name, batchSize, loggingFrequency, metrics) { private val logger = KotlinLogging.logger {} abstract fun prepareSql(sql: String): PreparedSql @@ -44,6 +47,8 @@ abstract class SqlDataLoader( batch: List, batchNo: Int ): BatchResult { + val timer = metrics.loadingDuration(name, groupId) + val failures = metrics.loadingFailures(name, groupId) val preparedSql = prepareSql(sqlUpsert) val duration = try { newSuspendedTransaction(db = database) { @@ -60,7 +65,9 @@ abstract class SqlDataLoader( } addBatch() } - executeBatch() + timer.recordInline { + executeBatch() + } } override fun prepareSQL(transaction: Transaction, prepared: Boolean): String = preparedSql.getSql() override fun arguments(): Iterable, Any?>>> = emptyList() @@ -72,7 +79,9 @@ abstract class SqlDataLoader( success = false, rowsLoaded = 0, errorMessage = "Error during loading data with loader $name: ${e.message ?: e.javaClass.simpleName}" - ) + ).also { + failures.increment() + } } return BatchResult( success = true, diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedAggregationTransformer.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedAggregationTransformer.kt index b4781aa04..abb6d3486 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedAggregationTransformer.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedAggregationTransformer.kt @@ -18,18 +18,18 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.DataTransformer import com.epam.drill.admin.etl.UntypedRow import com.epam.drill.admin.etl.flow.LruMap -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.trySendBlocking +import com.epam.drill.admin.etl.config.EtlMeter import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import mu.KotlinLogging -import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicInteger import kotlin.time.Duration.Companion.seconds class UntypedAggregationTransformer( override val name: String, private val bufferSize: Int, private val loggingFrequency: Int = 10, + private val metrics: EtlMeter, private val groupKeys: List, private val aggregate: (current: UntypedRow, next: UntypedRow) -> UntypedRow ) : DataTransformer { @@ -39,64 +39,58 @@ class UntypedAggregationTransformer( groupId: String, collector: Flow ): Flow = flow { - val transformedRows = AtomicLong(0) - val emittedRows = AtomicLong(0) - fun getAggregationRatio(): Double = - if (transformedRows.get() == 0L) 0.0 - else (1 - emittedRows.toDouble() / transformedRows.get()) - - val emittingChannel = Channel(capacity = bufferSize) - suspend fun drainChannel() { - var next = emittingChannel.tryReceive().getOrNull() - while (next != null) { - if (emittedRows.get() == 0L) - logger.debug { "ETL transformer [$name] for group [$groupId] started emitting aggregated rows..." } - emittedRows.incrementAndGet() - emit(next) - next = emittingChannel.tryReceive().getOrNull() - } - } - - val buffer = LruMap, UntypedRow>(maxSize = bufferSize) { _, value -> - emittingChannel.trySendBlocking(value) - } + var isTransformationStarted = false + val transformedRows = AtomicInteger() + val aggregatedRows = metrics.rowsAggregated(name, groupId) + val bufferOccupancy = metrics.aggregationBufferOccupancyRatio(name, groupId) + val buffer = LruMap, UntypedRow>(maxSize = bufferSize) trackProgressOf { try { collector.collect { row -> - if (transformedRows.get() == 0L) + if (!isTransformationStarted) { logger.debug { "ETL transformer [$name] for group [$groupId] started transformation..." } + isTransformationStarted = true + } val groupKey = groupKeys.map { row[it] } - buffer.compute(groupKey) { value -> + val evicted = buffer.compute(groupKey) { value -> if (value == null) { row } else { + aggregatedRows.increment() aggregate(value, row) } } - drainChannel() transformedRows.incrementAndGet() + bufferOccupancy.set(if (bufferSize == 0) 0.0 else buffer.size.toDouble() / bufferSize.toDouble()) + if (evicted != null) { + emit(evicted) + } } // Emit remaining aggregated rows - buffer.evictAll() - drainChannel() - } finally { - emittingChannel.close() + buffer.evictAll { _, evicted -> + bufferOccupancy.set(if (bufferSize == 0) 0.0 else buffer.size.toDouble() / bufferSize.toDouble()) + emit(evicted) + } + } catch (e: Exception) { + logger.error(e) { "ETL transformer [$name] for group [$groupId] failed during transformation: ${e.message}" } + throw e } }.every(loggingFrequency.seconds) { - if (transformedRows.get() > 0L) + if (isTransformationStarted) logger.debug { "ETL transformer [$name] for group [$groupId] transformed ${transformedRows.get()} rows" + - ", aggregation ratio: ${getAggregationRatio()}" + ", buffer occupancy: ${buffer.size}" + + ", buffer occupancy ratio: ${bufferOccupancy.get()}" } } - if (transformedRows.get() > 0L) { + if (isTransformationStarted) { logger.debug { - "ETL transformer [$name] for group [$groupId] completed transformation for $transformedRows rows, " + - "aggregation ratio: ${getAggregationRatio()}" + "ETL transformer [$name] for group [$groupId] completed transformation for $transformedRows rows" } } + bufferOccupancy.set(0.0) } } \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedFilterTransformer.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedFilterTransformer.kt new file mode 100644 index 000000000..353144b7a --- /dev/null +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedFilterTransformer.kt @@ -0,0 +1,47 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.etl.impl + +import com.epam.drill.admin.etl.DataTransformer +import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlMeter +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.flow + +class UntypedFilterTransformer( + override val name: String, + private val metrics: EtlMeter, + private val predicate: (UntypedRow) -> Boolean +) : DataTransformer { + override suspend fun transform( + groupId: String, + collector: Flow, + ): Flow { + val rowsFiltered = metrics.rowsFiltered(name, groupId) + return flow { + collector.filter { + predicate(it).also { passed -> + if (!passed) { + rowsFiltered.increment() + } + } + }.collect { row -> + emit(row) + } + } + } +} \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataExtractor.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataExtractor.kt index 308b85e38..ab1e804ff 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataExtractor.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataExtractor.kt @@ -16,6 +16,7 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlMeter import org.jetbrains.exposed.sql.Database import org.postgresql.util.PGobject import java.sql.ResultSet @@ -30,8 +31,9 @@ class UntypedSqlDataExtractor( fetchSize: Int = 2000, extractionLimit: Int = 1_000_000, loggingFrequency: Int = 10, + metrics: EtlMeter, private val lastExtractedAtColumnName: String, -) : SqlDataExtractor(name, extractionLimit, sqlQuery, database, fetchSize, loggingFrequency) { +) : SqlDataExtractor(name, extractionLimit, sqlQuery, database, fetchSize, loggingFrequency, metrics) { override fun prepareSql(sql: String): PreparedSql { return UntypedPreparedSql.prepareSql(sql) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataLoader.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataLoader.kt index bf038582c..5dcfc405e 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataLoader.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/UntypedSqlDataLoader.kt @@ -16,9 +16,8 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlMeter import org.jetbrains.exposed.sql.Database -import java.time.Instant -import java.util.Date class UntypedSqlDataLoader( name: String, @@ -27,14 +26,10 @@ class UntypedSqlDataLoader( database: Database, batchSize: Int = 1000, loggingFrequency: Int = 10, - val processable: (UntypedRow) -> Boolean = { true } -) : SqlDataLoader(name, batchSize, loggingFrequency, sqlUpsert, sqlDelete, database) { + metrics: EtlMeter, +) : SqlDataLoader(name, batchSize, loggingFrequency, sqlUpsert, sqlDelete, database, metrics) { override fun prepareSql(sql: String): PreparedSql { return UntypedPreparedSql.prepareSql(sql) } - - override fun isProcessable(args: UntypedRow): Boolean { - return processable(args) - } } \ No newline at end of file diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/BuildsEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/BuildsEtl.kt similarity index 86% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/BuildsEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/BuildsEtl.kt index 10ca1814c..3ed55410e 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/BuildsEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/BuildsEtl.kt @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -32,6 +32,7 @@ val EtlConfig.buildsExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "updated_at", + metrics = metrics, ) val EtlConfig.buildsLoader @@ -42,12 +43,10 @@ val EtlConfig.buildsLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, + metrics = metrics, ) val EtlConfig.buildsPipeline - get() = EtlPipelineImpl.singleLoader( - name = "builds", - extractor = buildsExtractor, - loader = buildsLoader, - bufferSize = bufferSize - ) \ No newline at end of file + get() = pipeline("builds") + .extractWith(buildsExtractor) + .loadWith(buildsLoader) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/CoverageEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/CoverageEtl.kt similarity index 57% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/CoverageEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/CoverageEtl.kt index aff6bf3cd..7d90f7799 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/CoverageEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/CoverageEtl.kt @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline import com.epam.drill.admin.etl.UntypedRow -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedAggregationTransformer +import com.epam.drill.admin.etl.impl.UntypedFilterTransformer import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader -import com.epam.drill.admin.etl.untypedNopTransformer import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig import org.postgresql.util.PGobject - val EtlConfig.coverageExtractor get() = UntypedSqlDataExtractor( name = "coverage", @@ -36,6 +35,7 @@ val EtlConfig.coverageExtractor fetchSize = fetchSize, extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, + metrics = metrics, lastExtractedAtColumnName = "created_at", ) @@ -48,6 +48,7 @@ val EtlConfig.testLaunchCoverageExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "test_completed_at", + metrics = metrics, ) val EtlConfig.buildMethodTestDefinitionCoverageLoader @@ -58,7 +59,7 @@ val EtlConfig.buildMethodTestDefinitionCoverageLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, - processable = { it["test_session_id"] != null && it["test_definition_id"] != null } + metrics = metrics, ) val EtlConfig.buildMethodTestSessionCoverageLoader @@ -69,15 +70,62 @@ val EtlConfig.buildMethodTestSessionCoverageLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, - processable = { it["test_session_id"] != null } + metrics = metrics, + ) + +val EtlConfig.buildMethodCoverageLoader + get() = UntypedSqlDataLoader( + name = "build_method_coverage", + sqlUpsert = fromResource("/etl/db/metrics/build_method_coverage_loader.sql"), + sqlDelete = fromResource("/etl/db/metrics/build_method_coverage_delete.sql"), + database = MetricsDatabaseConfig.database, + batchSize = batchSize, + loggingFrequency = loggingFrequency, + metrics = metrics, ) +val EtlConfig.methodDailyCoverageLoader + get() = UntypedSqlDataLoader( + name = "method_daily_coverage", + sqlUpsert = fromResource("/etl/db/metrics/method_daily_coverage_loader.sql"), + sqlDelete = fromResource("/etl/db/metrics/method_daily_coverage_delete.sql"), + database = MetricsDatabaseConfig.database, + batchSize = batchSize, + loggingFrequency = loggingFrequency, + metrics = metrics, + ) -val EtlConfig.buildMethodCoverageTransformer +val EtlConfig.test2CodeMappingLoader + get() = UntypedSqlDataLoader( + name = "test_to_code_mapping", + sqlUpsert = fromResource("/etl/db/metrics/test_to_code_mapping_loader.sql"), + sqlDelete = fromResource("/etl/db/metrics/test_to_code_mapping_delete.sql"), + database = MetricsDatabaseConfig.database, + batchSize = batchSize, + loggingFrequency = loggingFrequency, + metrics = metrics, + ) + +val EtlConfig.hasTestSessionFilter + get() = UntypedFilterTransformer( + name = "test_session_filter", + metrics = metrics, + predicate = { it["test_session_id"] != null }, + ) + +val EtlConfig.hasTestSessionAndDefinitionFilter + get() = UntypedFilterTransformer( + name = "test_definition_filter", + metrics = metrics, + predicate = { it["test_session_id"] != null && it["test_definition_id"] != null }, + ) + +val EtlConfig.buildMethodCoverageAggregator get() = UntypedAggregationTransformer( - name = "build_method_coverage", + name = "build_method_coverage_aggregator", bufferSize = transformationBufferSize, loggingFrequency = loggingFrequency, + metrics = metrics, groupKeys = listOf( "group_id", "app_id", @@ -86,31 +134,22 @@ val EtlConfig.buildMethodCoverageTransformer "app_env_id", "test_result", "test_tag", - "test_task_id", - ) - ) { current, next -> - val map = HashMap(current) - map["probes"] = mergeProbes(current["probes"], next["probes"]) - map["created_at_day"] = next["created_at_day"] - UntypedRow(next.timestamp, map) - } - - -val EtlConfig.buildMethodCoverageLoader - get() = UntypedSqlDataLoader( - name = "build_method_coverage", - sqlUpsert = fromResource("/etl/db/metrics/build_method_coverage_loader.sql"), - sqlDelete = fromResource("/etl/db/metrics/build_method_coverage_delete.sql"), - database = MetricsDatabaseConfig.database, - batchSize = batchSize, - loggingFrequency = loggingFrequency, + "test_task_id" + ), + aggregate = { current, next -> + val map = HashMap(current) + map["probes"] = mergeProbes(current["probes"], next["probes"]) + map["created_at_day"] = next["created_at_day"] + UntypedRow(next.timestamp, map) + }, ) -val EtlConfig.methodDailyCoverageTransformer +val EtlConfig.methodDailyCoverageAggregator get() = UntypedAggregationTransformer( - name = "method_daily_coverage", + name = "method_daily_coverage_aggregator", bufferSize = transformationBufferSize, loggingFrequency = loggingFrequency, + metrics = metrics, groupKeys = listOf( "group_id", "app_id", @@ -121,81 +160,86 @@ val EtlConfig.methodDailyCoverageTransformer "test_result", "test_tag", "test_task_id" - ) - ) { current, next -> - val map = HashMap(current) - map["probes"] = mergeProbes(current["probes"], next["probes"]) - UntypedRow(next.timestamp, map) - } - -val EtlConfig.methodDailyCoverageLoader - get() = UntypedSqlDataLoader( - name = "method_daily_coverage", - sqlUpsert = fromResource("/etl/db/metrics/method_daily_coverage_loader.sql"), - sqlDelete = fromResource("/etl/db/metrics/method_daily_coverage_delete.sql"), - database = MetricsDatabaseConfig.database, - batchSize = batchSize, - loggingFrequency = loggingFrequency, + ), + aggregate = { current, next -> + val map = HashMap(current) + map["probes"] = mergeProbes(current["probes"], next["probes"]) + UntypedRow(next.timestamp, map) + }, ) -val EtlConfig.test2CodeMappingTransformer - get() = UntypedAggregationTransformer( - name = "test_to_code_mapping", - bufferSize = transformationBufferSize, - loggingFrequency = loggingFrequency, - groupKeys = listOf( +val EtlConfig.buildMethodTestSessionCoveragePipeline + get() = pipeline("build_method_test_session_coverage") + .extractWith(coverageExtractor) + .transformWith(hasTestSessionFilter) + .loadWith(buildMethodTestSessionCoverageLoader) + +val EtlConfig.buildMethodCoveragePipeline + get() = pipeline("build_method_coverage") + .extractWith(coverageExtractor) + .transformWith(buildMethodCoverageAggregator) + .loadWith(buildMethodCoverageLoader) + +val EtlConfig.methodDailyCoveragePipeline + get() = pipeline("method_daily_coverage") + .extractWith(coverageExtractor) + .transformWith(methodDailyCoverageAggregator) + .loadWith(methodDailyCoverageLoader) + +val EtlConfig.testSessionBuildsFromCoveragePipeline + get() = pipeline("test_session_builds_from_coverage") + .extractWith(coverageExtractor) + .transformWith(hasTestSessionFilter) + .loadWith(testSessionBuildsLoader) + +val EtlConfig.buildMethodTestDefinitionCoveragePipeline + get() = pipeline("build_method_test_definition_coverage") + .extractWith(testLaunchCoverageExtractor) + .transformWith(hasTestSessionAndDefinitionFilter) + .loadWith(buildMethodTestDefinitionCoverageLoader) + +val EtlConfig.buildMethodTestSessionCoverageFromTestLaunchesPipeline + get() = pipeline("build_method_test_session_coverage_from_test_launches") + .extractWith(testLaunchCoverageExtractor) + .transformWith(hasTestSessionFilter) + .loadWith(buildMethodTestSessionCoverageLoader) + +val EtlConfig.buildMethodCoverageFromTestLaunchesPipeline + get() = pipeline("build_method_coverage_from_test_launches") + .extractWith(testLaunchCoverageExtractor) + .transformWith(buildMethodCoverageAggregator) + .loadWith(buildMethodCoverageLoader) + +val EtlConfig.methodDailyCoverageFromTestLaunchesPipeline + get() = pipeline("method_daily_coverage_from_test_launches") + .extractWith(testLaunchCoverageExtractor) + .transformWith(methodDailyCoverageAggregator) + .loadWith(methodDailyCoverageLoader) + +val EtlConfig.test2CodeMappingPipeline + get() = pipeline("test_to_code_mapping") + .extractWith(testLaunchCoverageExtractor) + .filter { it["test_definition_id"] != null && it["test_result"] == "PASSED" } + .aggregateBy( "group_id", "app_id", "signature", "test_definition_id", "branch", "app_env_id", - "test_task_id", - ) - ) { current, next -> - val map = HashMap(current) - map["updated_at_day"] = next["created_at_day"] - UntypedRow(next.timestamp, map) - } - -val EtlConfig.test2CodeMappingLoader - get() = UntypedSqlDataLoader( - name = "test_to_code_mapping", - sqlUpsert = fromResource("/etl/db/metrics/test_to_code_mapping_loader.sql"), - sqlDelete = fromResource("/etl/db/metrics/test_to_code_mapping_delete.sql"), - database = MetricsDatabaseConfig.database, - batchSize = batchSize, - loggingFrequency = loggingFrequency, - processable = { it["test_definition_id"] != null && it["test_result"] == "PASSED" } - ) - -val EtlConfig.coveragePipeline - get() = EtlPipelineImpl( - name = "coverage", - extractor = coverageExtractor, - loaders = listOf( - untypedNopTransformer to buildMethodTestSessionCoverageLoader, - buildMethodCoverageTransformer to buildMethodCoverageLoader, - methodDailyCoverageTransformer to methodDailyCoverageLoader, - untypedNopTransformer to testSessionBuildsLoader - ), - bufferSize = bufferSize - ) + "test_task_id" + ) { current, next -> + val map = HashMap(current) + map["updated_at_day"] = next["created_at_day"] + UntypedRow(next.timestamp, map) + } + .loadWith(test2CodeMappingLoader) -val EtlConfig.testLaunchCoveragePipeline - get() = EtlPipelineImpl( - name = "test_launch_coverage", - extractor = testLaunchCoverageExtractor, - loaders = listOf( - untypedNopTransformer to buildMethodTestDefinitionCoverageLoader, - untypedNopTransformer to buildMethodTestSessionCoverageLoader, - buildMethodCoverageTransformer to buildMethodCoverageLoader, - methodDailyCoverageTransformer to methodDailyCoverageLoader, - test2CodeMappingTransformer to test2CodeMappingLoader, - untypedNopTransformer to testSessionBuildsLoader - ), - bufferSize = bufferSize - ) +val EtlConfig.testSessionBuildsFromTestLaunchesPipeline + get() = pipeline("test_session_builds_from_test_launches") + .extractWith(testLaunchCoverageExtractor) + .transformWith(hasTestSessionFilter) + .loadWith(testSessionBuildsLoader) internal fun mergeProbes(current: Any?, next: Any?): PGobject { if (current == null || next == null) { @@ -210,9 +254,7 @@ internal fun mergeProbes(current: Any?, next: Any?): PGobject { throw IllegalArgumentException("Cannot merge probes of different lengths: current=${currentProbes.length}, next=${nextProbes.length}") val mergedProbes = buildString(currentProbes.length) { for (i in 0 until currentProbes.length) { - val currentBit = currentProbes[i] - val nextBit = nextProbes[i] - append(if (currentBit == '1' || nextBit == '1') '1' else '0') + append(if (currentProbes[i] == '1' || nextProbes[i] == '1') '1' else '0') } } return PGobject().apply { diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/MethodsEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/MethodsEtl.kt similarity index 70% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/MethodsEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/MethodsEtl.kt index d4618f3d8..8efd030aa 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/MethodsEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/MethodsEtl.kt @@ -13,15 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline import com.epam.drill.admin.etl.UntypedRow -import com.epam.drill.admin.etl.impl.EtlPipelineImpl -import com.epam.drill.admin.etl.impl.UntypedAggregationTransformer import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader -import com.epam.drill.admin.etl.untypedNopTransformer import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -34,6 +32,7 @@ val EtlConfig.buildMethodsExtractor fetchSize = fetchSize, extractionLimit = extractionLimit, lastExtractedAtColumnName = "created_at", + metrics = metrics, ) val EtlConfig.buildMethodsLoader @@ -42,35 +41,29 @@ val EtlConfig.buildMethodsLoader sqlUpsert = fromResource("/etl/db/metrics/build_methods_loader.sql"), sqlDelete = fromResource("/etl/db/metrics/build_methods_delete.sql"), database = MetricsDatabaseConfig.database, - batchSize = batchSize + batchSize = batchSize, + metrics = metrics, ) -val EtlConfig.methodLoaderTransformer - get() = UntypedAggregationTransformer( - name = "methods", - bufferSize = transformationBufferSize, - groupKeys = listOf( - "group_id", - "app_id", - "method_id" - ) - ) { current, next -> - UntypedRow(next.timestamp, next) - } - val EtlConfig.methodsLoader get() = UntypedSqlDataLoader( name = "methods", sqlUpsert = fromResource("/etl/db/metrics/methods_loader.sql"), sqlDelete = fromResource("/etl/db/metrics/methods_delete.sql"), database = MetricsDatabaseConfig.database, - batchSize = batchSize + batchSize = batchSize, + metrics = metrics, ) +val EtlConfig.buildMethodsPipeline + get() = pipeline("build_methods") + .extractWith(buildMethodsExtractor) + .loadWith(buildMethodsLoader) + val EtlConfig.methodsPipeline - get() = EtlPipelineImpl( - name = "methods", - extractor = buildMethodsExtractor, - loaders = listOf(untypedNopTransformer to buildMethodsLoader, methodLoaderTransformer to methodsLoader), - bufferSize = bufferSize - ) \ No newline at end of file + get() = pipeline("methods") + .extractWith(buildMethodsExtractor) + .aggregateBy("group_id", "app_id", "method_id") { _, next -> + UntypedRow(next.timestamp, next) + } + .loadWith(methodsLoader) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestDefinitionsEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestDefinitionsEtl.kt similarity index 86% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestDefinitionsEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestDefinitionsEtl.kt index 5eb663844..33fc49bcd 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestDefinitionsEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestDefinitionsEtl.kt @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -32,6 +32,7 @@ val EtlConfig.testDefinitionsExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "updated_at", + metrics = metrics, ) val EtlConfig.testDefinitionsLoader @@ -42,12 +43,10 @@ val EtlConfig.testDefinitionsLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, + metrics = metrics, ) val EtlConfig.testDefinitionsPipeline - get() = EtlPipelineImpl.singleLoader( - name = "test_definitions", - extractor = testDefinitionsExtractor, - loader = testDefinitionsLoader, - bufferSize = bufferSize - ) + get() = pipeline("test_definitions") + .extractWith(testDefinitionsExtractor) + .loadWith(testDefinitionsLoader) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestLaunchesEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestLaunchesEtl.kt similarity index 86% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestLaunchesEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestLaunchesEtl.kt index 3f686177d..a483795c9 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestLaunchesEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestLaunchesEtl.kt @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -32,6 +32,7 @@ val EtlConfig.testLaunchesExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "created_at", + metrics = metrics, ) val EtlConfig.testLaunchesLoader @@ -42,12 +43,10 @@ val EtlConfig.testLaunchesLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, + metrics = metrics, ) val EtlConfig.testLaunchesPipeline - get() = EtlPipelineImpl.singleLoader( - name = "test_launches", - extractor = testLaunchesExtractor, - loader = testLaunchesLoader, - bufferSize = bufferSize - ) + get() = pipeline("test_launches") + .extractWith(testLaunchesExtractor) + .loadWith(testLaunchesLoader) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionBuildsEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionBuildsEtl.kt similarity index 83% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionBuildsEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionBuildsEtl.kt index a5f8e5729..055b265c2 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionBuildsEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionBuildsEtl.kt @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -32,6 +32,7 @@ val EtlConfig.testSessionBuildsExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "created_at", + metrics = metrics, ) val EtlConfig.testSessionBuildsLoader @@ -42,13 +43,11 @@ val EtlConfig.testSessionBuildsLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, - processable = { it["test_session_id"] != null } + metrics = metrics, ) val EtlConfig.testSessionBuildsPipeline - get() = EtlPipelineImpl.singleLoader( - name = "test_session_builds", - extractor = testSessionBuildsExtractor, - loader = testSessionBuildsLoader, - bufferSize = bufferSize - ) + get() = pipeline("test_session_builds") + .extractWith(testSessionBuildsExtractor) + .transformWith(hasTestSessionFilter) + .loadWith(testSessionBuildsLoader) diff --git a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionsEtl.kt b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionsEtl.kt similarity index 86% rename from admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionsEtl.kt rename to admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionsEtl.kt index 36b836386..0c3d4f17f 100644 --- a/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/metrics/TestSessionsEtl.kt +++ b/admin-etl/src/main/kotlin/com/epam/drill/admin/etl/pipeline/TestSessionsEtl.kt @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.epam.drill.admin.etl.metrics +package com.epam.drill.admin.etl.pipeline -import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.UntypedSqlDataExtractor import com.epam.drill.admin.etl.impl.UntypedSqlDataLoader import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.impl.pipeline import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.metrics.config.fromResource import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig @@ -32,6 +32,7 @@ val EtlConfig.testSessionsExtractor extractionLimit = extractionLimit, loggingFrequency = loggingFrequency, lastExtractedAtColumnName = "created_at", + metrics = metrics, ) val EtlConfig.testSessionsLoader @@ -42,12 +43,10 @@ val EtlConfig.testSessionsLoader database = MetricsDatabaseConfig.database, batchSize = batchSize, loggingFrequency = loggingFrequency, + metrics = metrics, ) val EtlConfig.testSessionsPipeline - get() = EtlPipelineImpl.singleLoader( - name = "test_sessions", - extractor = testSessionsExtractor, - loader = testSessionsLoader, - bufferSize = bufferSize - ) + get() = pipeline("test_sessions") + .extractWith(testSessionsExtractor) + .loadWith(testSessionsLoader) diff --git a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/ETLSimpleTest.kt b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/ETLSimpleTest.kt index a7136cc8b..c40a2af6f 100644 --- a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/ETLSimpleTest.kt +++ b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/ETLSimpleTest.kt @@ -1,4 +1,4 @@ -/** +/** * Copyright 2020 - 2022 EPAM Systems * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,13 @@ */ package com.epam.drill.admin.etl +import com.epam.drill.admin.etl.config.EtlMeter import com.epam.drill.admin.etl.impl.EtlPipelineImpl import com.epam.drill.admin.etl.impl.EtlOrchestratorImpl +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.FlowCollector +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -32,13 +35,15 @@ data class SimpleClass(val id: Int, val createdAt: Instant): EtlRow(createdAt) private const val SIMPLE_PIPELINE = "simple-pipeline" private const val SIMPLE_EXTRACTOR = "simple-extractor" private const val SIMPLE_LOADER = "simple-loader" +private const val SIMPLE_TRANSFORMER = "simple-transformer" private const val FAILING_LOADER = "failing-loader" class ETLSimpleTest { private val dataStore = mutableListOf() private val idSequence = AtomicInteger() + private val metrics: EtlMeter = EtlMeter(SimpleMeterRegistry()) private fun addNewRecords(count: Int) { - repeat(count) { i -> + repeat(count) { dataStore.add(SimpleClass(idSequence.incrementAndGet(), Instant.now())) } } @@ -52,7 +57,7 @@ class ETLSimpleTest { emitter: FlowCollector, onExtractingProgress: suspend (EtlExtractingResult) -> Unit ) { - return dataStore.filter { it.createdAt > sinceTimestamp }.forEach { emitter.emit(it) } + dataStore.filter { it.createdAt > sinceTimestamp }.forEach { emitter.emit(it) } } } @@ -86,6 +91,16 @@ class ETLSimpleTest { } } + inner class SimpleTransformer : DataTransformer { + override val name = SIMPLE_TRANSFORMER + override suspend fun transform( + groupId: String, + collector: Flow + ): Flow = flow { + collector.collect { emit(it) } + } + } + inner class FailingLoader : DataLoader { override val name = FAILING_LOADER override suspend fun load( @@ -110,44 +125,24 @@ class ETLSimpleTest { } inner class SimpleMetadataRepository : EtlMetadataRepository { - private var metadata = EtlMetadata( - groupId = "test-group", - pipelineName = SIMPLE_PIPELINE, - lastProcessedAt = Instant.EPOCH, - lastRunAt = Instant.EPOCH, - status = EtlStatus.SUCCESS, - extractorName = SIMPLE_EXTRACTOR, - loaderName = SIMPLE_LOADER - ) + private val store = mutableMapOf() + + private fun key(groupId: String, pipelineName: String, extractorName: String, loaderName: String) = + "$groupId|$pipelineName|$extractorName|$loaderName" override suspend fun getMetadata( groupId: String, pipelineName: String, extractorName: String, loaderName: String - ): EtlMetadata? = metadata.copy( - groupId = groupId, - pipelineName = pipelineName, - extractorName = extractorName, - loaderName = loaderName - ) + ): EtlMetadata? = store[key(groupId, pipelineName, extractorName, loaderName)] override suspend fun saveMetadata(metadata: EtlMetadata) { - this@SimpleMetadataRepository.metadata = metadata + store[key(metadata.groupId, metadata.pipelineName, metadata.extractorName, metadata.loaderName)] = metadata } override suspend fun deleteMetadataByPipeline(groupId: String, pipelineName: String) { - if (metadata.pipelineName == pipelineName && metadata.groupId == groupId) { - metadata = EtlMetadata( - groupId = groupId, - pipelineName = pipelineName, - lastProcessedAt = Instant.EPOCH, - lastRunAt = Instant.EPOCH, - status = EtlStatus.SUCCESS, - extractorName = SIMPLE_EXTRACTOR, - loaderName = SIMPLE_LOADER - ) - } + store.keys.removeAll { it.startsWith("$groupId|$pipelineName|") } } override suspend fun getAllMetadataByExtractor( @@ -155,11 +150,12 @@ class ETLSimpleTest { pipelineName: String, extractorName: String ): List = - listOf(metadata).filter { it.groupId == groupId && it.extractorName == extractorName && it.pipelineName == pipelineName } + store.values.filter { + it.groupId == groupId && it.extractorName == extractorName && it.pipelineName == pipelineName + } - override suspend fun getAllMetadata(groupId: String): List { - return listOf(metadata).filter { it.groupId == groupId } - } + override suspend fun getAllMetadata(groupId: String): List = + store.values.filter { it.groupId == groupId } override suspend fun accumulateMetadataByLoader( groupId: String, @@ -172,11 +168,17 @@ class ETLSimpleTest { rowsProcessed: Long, errorMessage: String? ) { - this.metadata = this.metadata.copy( - lastProcessedAt = lastProcessedAt ?: this.metadata.lastProcessedAt, - lastLoadDuration = this.metadata.lastLoadDuration + loadDuration, - lastRowsProcessed = this.metadata.lastRowsProcessed + rowsProcessed, - status = status ?: (if (errorMessage != null) EtlStatus.FAILED else this.metadata.status), + val k = key(groupId, pipelineName, extractorName, loaderName) + val existing = store[k] ?: EtlMetadata( + groupId = groupId, pipelineName = pipelineName, extractorName = extractorName, + loaderName = loaderName, lastProcessedAt = Instant.EPOCH, lastRunAt = Instant.EPOCH, + status = EtlStatus.SUCCESS + ) + store[k] = existing.copy( + lastProcessedAt = lastProcessedAt ?: existing.lastProcessedAt, + lastLoadDuration = existing.lastLoadDuration + loadDuration, + lastRowsProcessed = existing.lastRowsProcessed + rowsProcessed, + status = status ?: (if (errorMessage != null) EtlStatus.FAILED else existing.status), errorMessage = errorMessage ) } @@ -189,25 +191,32 @@ class ETLSimpleTest { extractDuration: Long, errorMessage: String? ) { - this.metadata = this.metadata.copy( - lastExtractDuration = this.metadata.lastExtractDuration + extractDuration, - status = if (errorMessage != null) EtlStatus.FAILED else this.metadata.status, - errorMessage = errorMessage - ) + store.entries + .filter { (k, _) -> k.startsWith("$groupId|$pipelineName|$extractorName|") } + .forEach { (k, existing) -> + store[k] = existing.copy( + lastExtractDuration = existing.lastExtractDuration + extractDuration, + status = if (errorMessage != null) EtlStatus.FAILED else existing.status, + errorMessage = errorMessage + ) + } } } - val simpleOrchestrator = EtlOrchestratorImpl( - "simple-etl", - listOf( - EtlPipelineImpl.singleLoader( - "simple-pipeline", - extractor = SimpleExtractor(), - loader = SimpleLoader() - ) - ), - metadataRepository = SimpleMetadataRepository() - ) + private fun buildOrchestrator(metadataRepository: EtlMetadataRepository = SimpleMetadataRepository()) = + EtlOrchestratorImpl( + name = "simple-etl", + pipelines = listOf( + EtlPipelineImpl( + name = SIMPLE_PIPELINE, + extractor = SimpleExtractor(), + transformer = SimpleTransformer(), + loader = SimpleLoader(), + metrics = metrics, + ) + ), + metadataRepository = metadataRepository, + ) @BeforeEach fun setUp() { @@ -217,90 +226,74 @@ class ETLSimpleTest { @Test fun `given success loading, ETL orchestrator should move lastProcessedAt forward`() = runBlocking { - val orchestrator = simpleOrchestrator + val repo = SimpleMetadataRepository() + val orchestrator = buildOrchestrator(repo) val groupId = "test-group" - // Get initial lastProcessedAt timestamp - val initialMetadata = - orchestrator.metadataRepository.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, SIMPLE_LOADER)!! - val initialLastProcessedAt = initialMetadata.lastProcessedAt + val initialLastProcessedAt = repo.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, SIMPLE_LOADER) + ?.lastProcessedAt ?: Instant.EPOCH - // Add some data addNewRecords(3) - - // Run ETL val result = orchestrator.run(groupId) - // Verify ETL was successful assertTrue(result.first().status == EtlStatus.SUCCESS) assertEquals(3, result.first().rowsProcessed) - // Get updated metadata and verify lastProcessedAt moved forward - val updatedMetadata = - orchestrator.metadataRepository.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, SIMPLE_LOADER)!! - val updatedLastProcessedAt = updatedMetadata.lastProcessedAt + val updatedLastProcessedAt = repo.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, SIMPLE_LOADER) + ?.lastProcessedAt ?: Instant.EPOCH assertTrue(updatedLastProcessedAt > initialLastProcessedAt) - assertEquals(EtlStatus.SUCCESS, updatedMetadata.status) + assertEquals(EtlStatus.SUCCESS, repo.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, SIMPLE_LOADER)?.status) } @Test fun `given failed loading, ETL orchestrator should leave lastProcessedAt as initial`() = runBlocking { val groupId = "test-group" + val repo = SimpleMetadataRepository() val orchestrator = EtlOrchestratorImpl( - "failed-etl", - listOf( - EtlPipelineImpl.singleLoader( - "failed-pipeline", + name = "failed-etl", + pipelines = listOf( + EtlPipelineImpl( + name = "failed-pipeline", extractor = SimpleExtractor(), - loader = FailingLoader() + transformer = SimpleTransformer(), + loader = FailingLoader(), + metrics = metrics, ) ), - metadataRepository = SimpleMetadataRepository() + metadataRepository = repo, ) - // Get initial lastProcessedAt timestamp (should be EPOCH) - val initialMetadata = - orchestrator.metadataRepository.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, FAILING_LOADER)!! - val initialLastProcessedAt = initialMetadata.lastProcessedAt + val initialLastProcessedAt = repo.getMetadata(groupId, "failed-pipeline", SIMPLE_EXTRACTOR, FAILING_LOADER) + ?.lastProcessedAt ?: Instant.EPOCH - // Add some data addNewRecords(3) - - // Run ETL - should fail val result = orchestrator.run(groupId) - // Verify ETL failed assertTrue(result.first().status == EtlStatus.FAILED) assertEquals(0, result.first().rowsProcessed) - // Get updated metadata and verify lastProcessedAt remained unchanged - val updatedMetadata = - orchestrator.metadataRepository.getMetadata(groupId, SIMPLE_PIPELINE, SIMPLE_EXTRACTOR, FAILING_LOADER)!! - val updatedLastProcessedAt = updatedMetadata.lastProcessedAt + val updatedLastProcessedAt = repo.getMetadata(groupId, "failed-pipeline", SIMPLE_EXTRACTOR, FAILING_LOADER) + ?.lastProcessedAt ?: Instant.EPOCH assertEquals(initialLastProcessedAt, updatedLastProcessedAt) - assertEquals(EtlStatus.FAILED, updatedMetadata.status) + assertEquals(EtlStatus.FAILED, repo.getMetadata(groupId, "failed-pipeline", SIMPLE_EXTRACTOR, FAILING_LOADER)?.status) } @Test fun `given several launches, ETL orchestrator should process only new data`() = runBlocking { - val orchestrator = simpleOrchestrator + val repo = SimpleMetadataRepository() + val orchestrator = buildOrchestrator(repo) val groupId = "test-group" - // Add initial data addNewRecords(5) - // First run ETL — should process all initial data val result1 = orchestrator.run(groupId) assertTrue(result1.first().status == EtlStatus.SUCCESS) assertEquals(5, result1.first().rowsProcessed) Thread.sleep(10) - // Add new data after last processed timestamp addNewRecords(3) - // Second run ETL — should process only new data val result2 = orchestrator.run(groupId) - println(result2.first().errorMessage) assertTrue(result2.first().status == EtlStatus.SUCCESS) assertEquals(3, result2.first().rowsProcessed) } @@ -308,32 +301,33 @@ class ETLSimpleTest { @Test fun `given consistencyWindow, ETL orchestrator should re-process records within the lookback window`() = runBlocking { val groupId = "test-group" + val repo = SimpleMetadataRepository() val orchestrator = EtlOrchestratorImpl( name = "lookback-etl", pipelines = listOf( - EtlPipelineImpl.singleLoader( - "simple-pipeline", + EtlPipelineImpl( + name = SIMPLE_PIPELINE, extractor = SimpleExtractor(), - loader = SimpleLoader() + transformer = SimpleTransformer(), + loader = SimpleLoader(), + metrics = metrics, ) ), - metadataRepository = SimpleMetadataRepository(), - consistencyWindow = 60 + metadataRepository = repo, + consistencyWindow = 60, ) - // Add initial data addNewRecords(5) - // First run ETL — should process all initial data val result1 = orchestrator.run(groupId) assertTrue(result1.first().status == EtlStatus.SUCCESS) assertEquals(5, result1.first().rowsProcessed) - // Add new data after last processed timestamp addNewRecords(3) - // Second run ETL — lookback of 60s should re-process all 8 records (5 original + 3 new) - // because all records were created within the last 60 seconds + // lookback of 60s should re-process all 8 records (5 original + 3 new) val result2 = orchestrator.run(groupId) assertTrue(result2.first().status == EtlStatus.SUCCESS) assertEquals(8, result2.first().rowsProcessed) } } + + diff --git a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoaderTest.kt b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoaderTest.kt index 65ea6ed5d..24fc609a6 100644 --- a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoaderTest.kt +++ b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/BatchDataLoaderTest.kt @@ -17,6 +17,9 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.EtlRow import com.epam.drill.admin.etl.EtlStatus +import com.epam.drill.admin.etl.config.EtlMeter +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking import java.time.Instant @@ -32,12 +35,11 @@ class BatchDataLoaderTest { private val failOnBatch: Int = -1, private val failOnTimestamp: Boolean = false, private val outOfOrder: Boolean = false - ) : BatchDataLoader("test-loader", batchSize) { + ) : BatchDataLoader("test-loader", batchSize, + metrics = EtlMeter(SimpleMeterRegistry())) { val loadedBatches = mutableListOf>() - override fun isProcessable(args: TestItem): Boolean = args.processable - override suspend fun loadBatch(groupId: String, batch: List, batchNo: Int): BatchResult { if (failOnBatch == batchNo) { return BatchResult(success = false, rowsLoaded = 0, errorMessage = "Batch $batchNo failed") @@ -98,24 +100,6 @@ class BatchDataLoaderTest { assertEquals(EtlStatus.SUCCESS, results.last()) } - @Test - fun `load should skip non-processable items`() = runBlocking { - val items = listOf( - TestItem(Instant.ofEpochSecond(1), "item1"), - TestItem(Instant.ofEpochSecond(2), "item2", processable = false), - TestItem(Instant.ofEpochSecond(3), "item3") - ) - val loader = TestBatchDataLoader(batchSize = 10) - val result = loader.load("test-group", Instant.EPOCH, Instant.now(), flowOf(*items.toTypedArray())) { } - - assertEquals(false, result.isFailed) - assertEquals(2, result.processedRows) - assertEquals(1, loader.loadedBatches.size) - assertEquals(2, loader.loadedBatches[0].size) - assertEquals("item1", loader.loadedBatches[0][0].data) - assertEquals("item3", loader.loadedBatches[0][1].data) - } - @Test fun `load should fail on batch processing error`() = runBlocking { val items = (1..15).map { TestItem(Instant.ofEpochSecond(it.toLong()), "item$it") } diff --git a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractorTest.kt b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractorTest.kt index f9a5c485b..0bc433d40 100644 --- a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractorTest.kt +++ b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PageDataExtractorTest.kt @@ -17,7 +17,9 @@ package com.epam.drill.admin.etl.impl import com.epam.drill.admin.etl.EtlExtractingResult import com.epam.drill.admin.etl.EtlRow +import com.epam.drill.admin.etl.config.EtlMeter import com.epam.drill.admin.etl.impl.PageDataExtractorTest.TestPageDataExtractor.ExtractedPageInfo +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import kotlinx.coroutines.flow.FlowCollector import kotlinx.coroutines.runBlocking import java.time.Instant @@ -32,7 +34,8 @@ class PageDataExtractorTest { private class TestPageDataExtractor( extractionLimit: Int, private val data: List = emptyList(), - ) : PageDataExtractor("test-extractor", extractionLimit) { + ) : PageDataExtractor("test-extractor", extractionLimit, + metrics = EtlMeter(SimpleMeterRegistry())) { val extractedPages = mutableListOf() diff --git a/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilderTest.kt b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilderTest.kt new file mode 100644 index 000000000..e97eaf429 --- /dev/null +++ b/admin-etl/src/test/kotlin/com/epam/drill/admin/etl/impl/PipelineBuilderTest.kt @@ -0,0 +1,272 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.etl.impl + +import com.epam.drill.admin.etl.DataExtractor +import com.epam.drill.admin.etl.DataLoader +import com.epam.drill.admin.etl.DataTransformer +import com.epam.drill.admin.etl.EtlExtractingResult +import com.epam.drill.admin.etl.EtlLoadingResult +import com.epam.drill.admin.etl.EtlStatus +import com.epam.drill.admin.etl.UntypedRow +import com.epam.drill.admin.etl.config.EtlConfig +import com.epam.drill.admin.etl.config.EtlMeter +import io.ktor.server.config.MapApplicationConfig +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.FlowCollector +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test +import java.time.Instant +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + + +class ExtractStepTest { + + @Test + fun `loadWith delivers all rows to the loader`() { + val loader = PbCapturingLoader("loader") + val rows = listOf(pbRow(1), pbRow(2), pbRow(3)) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(3, loader.received.size) + } + + @Test + fun `filter on ExtractStep keeps only matching rows`() { + val loader = PbCapturingLoader("loader") + val rows = listOf(pbRow(1), pbRow(2), pbRow(3), pbRow(4)) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .filter { (it["value"] as Int) % 2 == 0 } + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(2, loader.received.size) + assertTrue(loader.received.all { (it["value"] as Int) % 2 == 0 }) + } + + @Test + fun `transformWith on ExtractStep applies the transformer before loading`() { + val loader = PbCapturingLoader("loader") + val rows = listOf(pbRow(10), pbRow(20)) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .transformWith(PbPassThroughTransformer()) + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(2, loader.received.size) + assertEquals(listOf(10, 20), loader.received.map { it["value"] }) + } + + @Test + fun `aggregateBy on ExtractStep sums values for the same key`() { + val loader = PbCapturingLoader("loader") + val rows = listOf( + UntypedRow(Instant.now(), mapOf("k" to "A", "v" to 1)), + UntypedRow(Instant.now(), mapOf("k" to "A", "v" to 2)), + UntypedRow(Instant.now(), mapOf("k" to "B", "v" to 5)), + ) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .aggregateBy("k") { current, next -> + UntypedRow( + next.timestamp, + mapOf("k" to current["k"], "v" to (current["v"] as Int) + (next["v"] as Int)) + ) + } + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(2, loader.received.size) + val sumA = loader.received.first { it["k"] == "A" }["v"] as Int + assertEquals(3, sumA) + } +} + +class TransformStepTest { + + @Test + fun `filter on TransformStep narrows rows after the transformer`() { + val loader = PbCapturingLoader("loader") + val rows = listOf(pbRow(1), pbRow(2), pbRow(3), pbRow(4), pbRow(5)) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .transformWith(PbPassThroughTransformer()) + .filter { (it["value"] as Int) > 3 } + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(2, loader.received.size) + assertTrue(loader.received.all { (it["value"] as Int) > 3 }) + } + + @Test + fun `chained filter calls on TransformStep compose with logical AND semantics`() { + val loader = PbCapturingLoader("loader") + val rows = (1..6).map { pbRow(it) } + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .filter { (it["value"] as Int) > 2 } + .filter { (it["value"] as Int) < 5 } + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(2, loader.received.size) + assertTrue(loader.received.all { (it["value"] as Int) in 3..4 }) + } + + @Test + fun `transformWith on TransformStep chains transformers correctly`() { + val loader = PbCapturingLoader("loader") + val rows = listOf(pbRow(1), pbRow(2), pbRow(3)) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .transformWith(PbPassThroughTransformer("t1")) + .transformWith(PbPassThroughTransformer("t2")) + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(3, loader.received.size) + } + + @Test + fun `aggregateBy on TransformStep aggregates after a prior transform`() { + val loader = PbCapturingLoader("loader") + val rows = listOf( + UntypedRow(Instant.now(), mapOf("k" to "X", "v" to 10)), + UntypedRow(Instant.now(), mapOf("k" to "X", "v" to 20)), + ) + + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(rows)) + .transformWith(PbPassThroughTransformer()) + .aggregateBy("k") { cur, nxt -> + UntypedRow( + nxt.timestamp, + mapOf("k" to cur["k"], "v" to (cur["v"] as Int) + (nxt["v"] as Int)) + ) + } + .loadWith(loader) + + pbExecute(pipeline) + + assertEquals(1, loader.received.size) + assertEquals(30, loader.received[0]["v"] as Int) + } + + @Test + fun `loadWith on TransformStep produces a pipeline with a single loader`() { + val loader = PbCapturingLoader("loader") + val pipeline = pbCfg.pipeline("t") + .extractWith(PbFixedExtractor(emptyList())) + .transformWith(PbPassThroughTransformer()) + .loadWith(loader) + + assertNotNull(pipeline.loader) + assertEquals(loader.name, pipeline.loader.name) + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +private val pbCfg = EtlConfig( + config = MapApplicationConfig(), + metrics = EtlMeter(SimpleMeterRegistry()), +) + +private fun pbRow(value: Any?, key: String = "value", ts: Instant = Instant.now()) = + UntypedRow(ts, mapOf(key to value)) + +private fun pbExecute( + pipeline: com.epam.drill.admin.etl.EtlPipeline, +) = runBlocking { + // Use an empty flow — tests only verify pipeline structure and transformation logic via pbExecute + // through the orchestrator in integration tests; here we call execute with a pre-built extractor flow. + val rows = mutableListOf() + // Re-extract via the pipeline's extractor into a simple list, then replay as flow + pipeline.extractor.extract("g1", Instant.EPOCH, Instant.now(), { rows.add(it) }, {}) + val extractedFlow = kotlinx.coroutines.flow.flow { rows.forEach { emit(it) } } + pipeline.execute( + groupId = "g1", + sinceTimestamp = Instant.EPOCH, + untilTimestamp = Instant.now(), + extractedFlow = extractedFlow, + ) +} + +private class PbFixedExtractor( + private val rows: List, + override val name: String = "fixed", +) : DataExtractor { + override suspend fun extract( + groupId: String, + sinceTimestamp: Instant, + untilTimestamp: Instant, + emitter: FlowCollector, + onExtractingProgress: suspend (EtlExtractingResult) -> Unit, + ) = rows.forEach { emitter.emit(it) } +} + +private class PbCapturingLoader(override val name: String) : DataLoader { + val received = mutableListOf() + + override suspend fun load( + groupId: String, + sinceTimestamp: Instant, + untilTimestamp: Instant, + collector: Flow, + onLoadingProgress: suspend (EtlLoadingResult) -> Unit, + onStatusChanged: suspend (EtlStatus) -> Unit, + ): EtlLoadingResult { + collector.collect { received.add(it) } + return EtlLoadingResult(lastProcessedAt = untilTimestamp, processedRows = received.size.toLong()) + } + + override suspend fun deleteAll(groupId: String) = received.clear() +} + +private class PbPassThroughTransformer(override val name: String = "pass") : + DataTransformer { + override suspend fun transform(groupId: String, collector: Flow): Flow = flow { + collector.collect { emit(it) } + } +} + + diff --git a/admin-metrics/build.gradle.kts b/admin-metrics/build.gradle.kts index 945ec1f7d..8ca4e74fe 100644 --- a/admin-metrics/build.gradle.kts +++ b/admin-metrics/build.gradle.kts @@ -16,12 +16,14 @@ val ktorVersion: String by parent!!.extra val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val mockitoKotlinVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra val flywaydbVersion: String by parent!!.extra val postgresSqlVersion: String by parent!!.extra val zaxxerHikaricpVersion: String by parent!!.extra val quartzVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -59,11 +61,12 @@ dependencies { implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") implementation("org.quartz-scheduler:quartz:$quartzVersion") + implementation("io.micrometer:micrometer-core:${micrometerVersion}") api("org.flywaydb:flyway-core:$flywaydbVersion") compileOnly("org.postgresql:postgresql:$postgresSqlVersion") testImplementation("org.jetbrains.kotlin:kotlin-test:$kotlinVersion") - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") testImplementation("io.ktor:ktor-server-test-host:$ktorVersion") testImplementation("io.ktor:ktor-client-resources:$ktorVersion") testImplementation("com.jayway.jsonpath:json-path:2.9.0") diff --git a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/BuildsInfoApiTest.kt b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/BuildsInfoApiTest.kt index 62212a28a..f5c813bcd 100644 --- a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/BuildsInfoApiTest.kt +++ b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/BuildsInfoApiTest.kt @@ -18,6 +18,7 @@ package com.epam.drill.admin.metrics import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.test.* import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig +import com.epam.drill.admin.writer.rawdata.route.payload.BuildInfoPayload import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload import com.epam.drill.admin.writer.rawdata.route.payload.InstancePayload import com.epam.drill.admin.writer.rawdata.table.BuildTable @@ -36,11 +37,11 @@ class BuildsInfoApiTest : MetricsDatabaseTests({ default, metrics -> MetricsDatabaseConfig.init(metrics) }) { private suspend fun TestDataDsl.initTestData() { - client.putBuild(BuildPayload(groupId = testGroup, appId = testApp, buildVersion = "1.0.0", branch = testBranch)) - client.putBuild(BuildPayload(groupId = testGroup, appId = testApp, buildVersion = "2.0.0", branch = testBranch)) - client.putBuild(BuildPayload(groupId = testGroup, appId = testApp, buildVersion = "3.0.0", branch = "develop")) - client.putBuild(BuildPayload(groupId = testGroup, appId = "app-2", buildVersion = "1.0.0", branch = testBranch)) - client.putBuild(BuildPayload(groupId = "group-2", appId = testApp, buildVersion = "1.0.0", branch = testBranch)) + client.putBuildInfo(BuildInfoPayload(groupId = testGroup, appId = testApp, buildVersion = "1.0.0", branch = testBranch)) + client.putBuildInfo(BuildInfoPayload(groupId = testGroup, appId = testApp, buildVersion = "2.0.0", branch = testBranch)) + client.putBuildInfo(BuildInfoPayload(groupId = testGroup, appId = testApp, buildVersion = "3.0.0", branch = "develop")) + client.putBuildInfo(BuildInfoPayload(groupId = testGroup, appId = "app-2", buildVersion = "1.0.0", branch = testBranch)) + client.putBuildInfo(BuildInfoPayload(groupId = "group-2", appId = testApp, buildVersion = "1.0.0", branch = testBranch)) } private suspend fun TestDataDsl.initEnvironmentData() { diff --git a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageTreemapTest.kt b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageTreemapTest.kt index c7980c9e6..997a9f8df 100644 --- a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageTreemapTest.kt +++ b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageTreemapTest.kt @@ -19,6 +19,7 @@ import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.test.MetricsDatabaseTests import com.epam.drill.admin.test.withTransaction import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig +import com.epam.drill.admin.writer.rawdata.route.payload.BuildInfoPayload import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload import com.epam.drill.admin.writer.rawdata.route.payload.InstancePayload import com.epam.drill.admin.writer.rawdata.route.payload.SingleMethodPayload @@ -35,7 +36,7 @@ class CoverageTreemapTest : MetricsDatabaseTests({ default, metrics -> }) { @Test fun `given build with no methods no coverage, coverage-treemap should return empty list`() = havingData { - client.putBuild(BuildPayload(groupId = testGroup, appId = testApp, buildVersion = "1.0.0", branch = "main")) + client.putBuildInfo(BuildInfoPayload(groupId = testGroup, appId = testApp, buildVersion = "1.0.0", branch = "main")) }.expectThat { client.get("/metrics/coverage-treemap") { parameter("buildId", "${testGroup}:${testApp}:1.0.0") diff --git a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/DataIngestClient.kt b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/DataIngestClient.kt index 5ff674e01..611ea0b4f 100644 --- a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/DataIngestClient.kt +++ b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/DataIngestClient.kt @@ -15,7 +15,6 @@ */ package com.epam.drill.admin.metrics -import com.epam.drill.admin.writer.rawdata.config.toBitString import com.epam.drill.admin.writer.rawdata.route.payload.* import com.jayway.jsonpath.JsonPath import io.ktor.client.* @@ -100,6 +99,12 @@ val TestDetails.definitionId: String } suspend fun HttpClient.putBuild(payload: BuildPayload): HttpResponse { + return put("/data-ingest/builds") { + setBody(payload) + }.assertSuccessStatus() +} + +suspend fun HttpClient.putBuildInfo(payload: BuildInfoPayload): HttpResponse { return put("/data-ingest/builds/info") { setBody(payload) }.assertSuccessStatus() diff --git a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/RecommendedTestsApiTest.kt b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/RecommendedTestsApiTest.kt index be4566a0d..89c2e4cae 100644 --- a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/RecommendedTestsApiTest.kt +++ b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/RecommendedTestsApiTest.kt @@ -19,6 +19,7 @@ import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig import com.epam.drill.admin.test.MetricsDatabaseTests import com.epam.drill.admin.test.withTransaction import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig +import com.epam.drill.admin.writer.rawdata.route.payload.BuildInfoPayload import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload import com.epam.drill.admin.writer.rawdata.table.* import io.ktor.client.request.* @@ -140,8 +141,8 @@ class RecommendedTestsApiTest : MetricsDatabaseTests({ default, metrics -> @Test fun `given baselineBuildBranches parameter, recommended test service should suggest skipping tests if they are not impacted in baselines from specified branch`() { havingData { - client.putBuild(BuildPayload(groupId = build1.groupId, appId = build1.appId, buildVersion = build1.buildVersion, branch = "main")) - client.putBuild(BuildPayload(groupId = build2.groupId, appId = build2.appId, buildVersion = build2.buildVersion, branch = "feature")) + client.putBuildInfo(BuildInfoPayload(groupId = build1.groupId, appId = build1.appId, buildVersion = build1.buildVersion, branch = "main")) + client.putBuildInfo(BuildInfoPayload(groupId = build2.groupId, appId = build2.appId, buildVersion = build2.buildVersion, branch = "feature")) //build1 on main branch, test1 covers method2 client.deployInstance(build1, arrayOf(method1, method2)) client.launchTest( diff --git a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/TestDataDsl.kt b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/TestDataDsl.kt index 4c3a56274..d5a2c6bbf 100644 --- a/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/TestDataDsl.kt +++ b/admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/TestDataDsl.kt @@ -26,6 +26,7 @@ import com.epam.drill.admin.metrics.views.TestImpactStatus import com.epam.drill.admin.test.StubDrillScheduler import com.epam.drill.admin.test.drillApplication import com.epam.drill.admin.test.drillClient +import com.epam.drill.admin.test.waitUntilInBlocking import com.epam.drill.admin.writer.rawdata.config.rawDataServicesDIModule import com.epam.drill.admin.writer.rawdata.route.dataIngestRoutes import com.epam.drill.admin.writer.rawdata.route.payload.InstancePayload @@ -43,12 +44,14 @@ import org.kodein.di.instance import org.kodein.di.singleton val scheduler = DI.Module("testModule") { - bind() with singleton { - StubDrillScheduler(UpdateMetricsEtlJob( + bind() with singleton { + StubDrillScheduler( + UpdateMetricsEtlJob( instance(), instance() - )) - } + ) + ) } +} fun havingData(testsData: suspend TestDataDsl.() -> Unit): HttpClient { return runBlocking { @@ -214,7 +217,9 @@ class TestDataDsl(val client: HttpClient) { fun HttpClient.expectThat(checks: suspend ExpectationDsl.(HttpClient) -> Unit) { val client = this - return runBlocking { + return waitUntilInBlocking( + onAssertionFailed = { refreshMetrics() } + ) { checks(ExpectationDsl(client), client) } } diff --git a/admin-test/build.gradle.kts b/admin-test/build.gradle.kts index c9d2d8b88..2dc7081fe 100644 --- a/admin-test/build.gradle.kts +++ b/admin-test/build.gradle.kts @@ -16,6 +16,7 @@ val ktorVersion: String by parent!!.extra val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val mockitoKotlinVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra val flywaydbVersion: String by parent!!.extra @@ -24,6 +25,8 @@ val junitJupiterVersion: String by parent!!.extra val postgresSqlVersion: String by parent!!.extra val zaxxerHikaricpVersion: String by parent!!.extra val quartzVersion: String by parent!!.extra +val awaitilityVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -65,8 +68,10 @@ dependencies { implementation("org.junit.jupiter:junit-jupiter:$junitJupiterVersion") implementation("org.testcontainers:postgresql:$testContainersVersion") implementation("com.zaxxer:HikariCP:$zaxxerHikaricpVersion") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") implementation("org.postgresql:postgresql:$postgresSqlVersion") + implementation("org.awaitility:awaitility:${awaitilityVersion}") + implementation("io.micrometer:micrometer-core:${micrometerVersion}") } tasks { diff --git a/admin-test/src/main/kotlin/com/epam/drill/admin/test/AwaitDb.kt b/admin-test/src/main/kotlin/com/epam/drill/admin/test/AwaitDb.kt new file mode 100644 index 000000000..5a3e44b63 --- /dev/null +++ b/admin-test/src/main/kotlin/com/epam/drill/admin/test/AwaitDb.kt @@ -0,0 +1,60 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.test + +import kotlinx.coroutines.runBlocking +import org.awaitility.Awaitility.await +import org.jetbrains.exposed.sql.transactions.transaction +import java.time.Duration + +private val DEFAULT_DB_WAIT_TIMEOUT: Duration = Duration.ofSeconds(5) +private val DEFAULT_DB_POLL_INTERVAL: Duration = Duration.ofMillis(100) + +fun waitUntilInTransaction(assertion: () -> Unit) { + await() + .atMost(DEFAULT_DB_WAIT_TIMEOUT) + .pollInterval(DEFAULT_DB_POLL_INTERVAL) + .untilAsserted { + transaction { + assertion() + } + } +} + +fun waitUntilInBlocking( + onAssertionFailed: suspend (AssertionError) -> Unit = {}, + assertion: suspend () -> Unit +) { + await() + .atMost(DEFAULT_DB_WAIT_TIMEOUT) + .pollInterval(DEFAULT_DB_POLL_INTERVAL) + .untilAsserted { + runCatching { + runBlocking { + assertion() + } + }.onFailure { e -> + if (e is AssertionError) { + runCatching { + runBlocking { + onAssertionFailed(e) + } + } + } + throw e + } + } +} \ No newline at end of file diff --git a/admin-test/src/main/kotlin/com/epam/drill/admin/test/TestUtils.kt b/admin-test/src/main/kotlin/com/epam/drill/admin/test/TestUtils.kt index 6b673646c..cb31e9a92 100644 --- a/admin-test/src/main/kotlin/com/epam/drill/admin/test/TestUtils.kt +++ b/admin-test/src/main/kotlin/com/epam/drill/admin/test/TestUtils.kt @@ -16,7 +16,6 @@ package com.epam.drill.admin.test import io.ktor.serialization.kotlinx.json.* -import io.ktor.server.application.install import io.ktor.server.plugins.contentnegotiation.* import io.ktor.server.plugins.statuspages.StatusPages import io.ktor.server.resources.* @@ -29,8 +28,17 @@ import org.kodein.di.DI import org.kodein.di.ktor.di import kotlin.test.assertEquals import com.epam.drill.admin.common.route.commonStatusPages +import io.ktor.server.application.ApplicationStopping +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import org.jetbrains.exposed.sql.Database -import javax.sql.DataSource +import org.kodein.di.allInstances +import org.kodein.di.bind +import org.kodein.di.ktor.closestDI +import org.kodein.di.singleton +import kotlin.getValue fun withRollback(test: suspend () -> Unit) { @@ -55,7 +63,7 @@ fun withTransaction(db: Database? = null, test: suspend () -> Unit) { fun drillApplication( vararg diModules: DI.Module = emptyArray(), - routes: Routing.() -> Unit + routes: Route.() -> Unit ) = TestApplication { install(Resources) install(ContentNegotiation) { @@ -66,8 +74,19 @@ fun drillApplication( } application { di { + import(meterModule) diModules.forEach { import(it) } } + monitor.subscribe(ApplicationStopping) { + val closableComponents: List by closestDI().allInstances() + runBlocking { + closableComponents.map { + async { + it.close() + } + }.awaitAll() + } + } } routing { routes() @@ -90,4 +109,10 @@ fun assertJsonEquals(json1: String, json2: String) { val obj1: JsonElement = removeNulls(json.parseToJsonElement(json1)) val obj2: JsonElement = removeNulls(json.parseToJsonElement(json2)) assertEquals(obj1, obj2) +} + +private val meterModule = DI.Module("meterModule") { + bind() with singleton { + SimpleMeterRegistry() + } } \ No newline at end of file diff --git a/admin-writer/build.gradle.kts b/admin-writer/build.gradle.kts index b3c06a829..5f833af77 100644 --- a/admin-writer/build.gradle.kts +++ b/admin-writer/build.gradle.kts @@ -15,6 +15,7 @@ val ktorVersion: String by parent!!.extra val kodeinVersion: String by parent!!.extra val kotlinxSerializationVersion: String by parent!!.extra val kotlinxDatetimeVersion: String by parent!!.extra +val kotlinxCoroutinesVersion: String by parent!!.extra val mockitoKotlinVersion: String by parent!!.extra val exposedVersion: String by parent!!.extra val quartzVersion: String by parent!!.extra @@ -23,6 +24,9 @@ val testContainersVersion: String by parent!!.extra val postgresSqlVersion: String by parent!!.extra val zaxxerHikaricpVersion: String by parent!!.extra val logbackVersion: String by parent!!.extra +val kafkaClientsVersion: String by parent!!.extra +val junitJupiterVersion: String by parent!!.extra +val micrometerVersion: String by parent!!.extra repositories { mavenLocal() @@ -62,18 +66,24 @@ dependencies { api("org.flywaydb:flyway-core:$flywaydbVersion") compileOnly("org.postgresql:postgresql:$postgresSqlVersion") + implementation("org.apache.kafka:kafka-clients:$kafkaClientsVersion") + implementation("io.ktor:ktor-client-core:$ktorVersion") implementation("io.ktor:ktor-client-apache:$ktorVersion") - implementation("io.ktor:ktor-client-json:$ktorVersion") - implementation("io.ktor:ktor-client-serialization:$ktorVersion") implementation("io.ktor:ktor-client-content-negotiation:$ktorVersion") implementation("io.ktor:ktor-server-resources:$ktorVersion") implementation("io.ktor:ktor-server-status-pages:$ktorVersion") + implementation("io.micrometer:micrometer-core:${micrometerVersion}") testImplementation("org.jetbrains.kotlin:kotlin-test:$kotlinVersion") - testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$kotlinxCoroutinesVersion") testImplementation("io.ktor:ktor-server-test-host:$ktorVersion") testImplementation("ch.qos.logback:logback-classic:$logbackVersion") + testImplementation("org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion") + testImplementation("org.testcontainers:testcontainers:$testContainersVersion") + testImplementation("org.testcontainers:junit-jupiter:$testContainersVersion") + testImplementation("org.testcontainers:kafka:$testContainersVersion") + testImplementation("org.junit.jupiter:junit-jupiter:$junitJupiterVersion") testImplementation(project(":admin-test")) } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/KafkaConfig.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/KafkaConfig.kt new file mode 100644 index 000000000..3c66c15b7 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/KafkaConfig.kt @@ -0,0 +1,73 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.config + +import io.ktor.server.config.ApplicationConfig +import org.apache.kafka.clients.CommonClientConfigs +import org.apache.kafka.common.config.SaslConfigs +import org.apache.kafka.common.config.SslConfigs +import java.util.Properties + +/** + * Common Kafka cluster and connection configuration shared by all Kafka clients. + */ +class KafkaConfig(private val config: ApplicationConfig) { + + /** + * Comma-separated list of `host:port` pairs used to establish the initial connection to the + * Kafka cluster. + */ + val bootstrapServers: String + get() = config.string("bootstrapServers", "localhost:9092") + + /** + * Builds a [Properties] map containing connection settings shared by producer and consumer + * clients (DNS lookup, idle/backoff timeouts, and security/SSL properties). + */ + fun toCommonProperties(): Properties = Properties().apply { + put(CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG, config.string("clientDnsLookup", "use_all_dns_ips")) + put(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG, config.int("connectionsMaxIdleMs", 540_000).toString()) + put(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG, config.int("reconnectBackoffMs", 50).toString()) + put(CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG, config.int("reconnectBackoffMaxMs", 1_000).toString()) + put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, config.int("retryBackoffMs", 100).toString()) + config.stringOrNull("securityProtocol")?.let { put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, it) } + config.stringOrNull("saslMechanism")?.let { put(SaslConfigs.SASL_MECHANISM, it) } + config.stringOrNull("saslJaasConfig")?.let { put(SaslConfigs.SASL_JAAS_CONFIG, it) } + config.stringOrNull("sslTruststoreLocation")?.let { put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, it) } + config.stringOrNull("sslTruststorePassword")?.let { put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, it) } + config.stringOrNull("sslKeystoreLocation")?.let { put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, it) } + config.stringOrNull("sslKeystorePassword")?.let { put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, it) } + config.stringOrNull("sslKeyPassword")?.let { put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, it) } + config.stringOrNull("sslEndpointIdentificationAlgorithm") + ?.let { put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, it) } + } +} + +internal fun ApplicationConfig.string(path: String, default: String): String = + stringOrNull(path) ?: default + +internal fun ApplicationConfig.stringOrNull(path: String): String? = + propertyOrNull(path)?.getString()?.takeIf { it.isNotBlank() } + +internal fun ApplicationConfig.int(path: String, default: Int): Int = + propertyOrNull(path)?.getString()?.toIntOrNull() ?: default + +internal fun ApplicationConfig.long(path: String, default: Long): Long = + propertyOrNull(path)?.getString()?.toLongOrNull() ?: default + +internal fun ApplicationConfig.boolean(path: String, default: Boolean): Boolean = + propertyOrNull(path)?.getString()?.toBooleanStrictOrNull() ?: default + diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataKafkaQueueConfig.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataKafkaQueueConfig.kt new file mode 100644 index 000000000..90fd1bdcf --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataKafkaQueueConfig.kt @@ -0,0 +1,104 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.config + +import io.ktor.server.config.ApplicationConfig +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.producer.ProducerConfig +import java.util.Properties + +/** + * Raw-data-queue-specific Kafka configuration. + * + * Contains settings that are private to the raw-data processing task: topic name, + * consumer group, poll/shutdown timeouts, and producer/consumer tuning parameters. + * Cluster-level connection settings (bootstrap servers, security, SSL) are provided + * by the shared [KafkaConfig]. + * + * @param config Application config scoped to `drill.rawData.queue.kafka`. + * @param kafkaConfig Shared Kafka cluster and connection configuration. + */ +class RawDataKafkaQueueConfig( + private val config: ApplicationConfig, + private val kafkaConfig: KafkaConfig, +) { + /** + * Name of the Kafka topic to which raw data is produced and from which it is consumed. + * Auto creation of the topic is disabled by default, so it must be provisioned before starting the service. + */ + val topic: String + get() = config.string("topic", "drill-raw-data") + + /** + * Consumer group identifier shared by all consumer instances of this service. + */ + val consumerGroupId: String + get() = config.string("consumerGroupId", "drill-writer") + + /** + * Maximum time in milliseconds the consumer will block waiting for new records in a + * single [org.apache.kafka.clients.consumer.KafkaConsumer.poll] call. + */ + val pollTimeoutMs: Long + get() = config.long("pollTimeoutMs", 500L) + + /** + * Maximum time in milliseconds to wait for in-flight records to be flushed and the consumer + * to commit its offsets during a graceful shutdown. + * Should be greater than [pollTimeoutMs] to avoid data loss on shutdown. + */ + val shutdownTimeoutMs: Long + get() = config.long("shutdownTimeoutMs", 5_000L) + + /** + * Builds a [java.util.Properties] map for the Kafka producer. + * Common connection properties are contributed by [kafkaConfig]. + */ + val producerProperties: Properties + get() = Properties().apply { + putAll(kafkaConfig.toCommonProperties()) + put(ProducerConfig.CLIENT_ID_CONFIG, config.string("producer.clientId", "drill-admin-raw-data-producer")) + put(ProducerConfig.ACKS_CONFIG, config.string("producer.acks", "all")) + put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, config.boolean("producer.enableIdempotence", true).toString()) + put(ProducerConfig.RETRIES_CONFIG, config.int("producer.retries", Int.MAX_VALUE).toString()) + put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, config.int("producer.maxInFlightRequestsPerConnection", 5).toString()) + put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, config.int("producer.deliveryTimeoutMs", 120_000).toString()) + put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, config.int("producer.requestTimeoutMs", 30_000).toString()) + put(ProducerConfig.LINGER_MS_CONFIG, config.int("producer.lingerMs", 20).toString()) + put(ProducerConfig.BATCH_SIZE_CONFIG, config.int("producer.batchSize", 32_768).toString()) + put(ProducerConfig.COMPRESSION_TYPE_CONFIG, config.string("producer.compressionType", "lz4")) + } + + /** + * Builds a [java.util.Properties] map for the Kafka consumer. + * Common connection properties are contributed by [kafkaConfig]. + */ + val consumerProperties: Properties + get() = Properties().apply { + putAll(kafkaConfig.toCommonProperties()) + put(ConsumerConfig.CLIENT_ID_CONFIG, config.string("consumer.clientId", "drill-admin-raw-data-consumer")) + put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false") + put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, config.boolean("consumer.allowAutoCreateTopics", false).toString()) + put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, config.string("consumer.autoOffsetReset", "earliest")) + put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, config.string("consumer.isolationLevel", "read_committed")) + put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, config.int("consumer.maxPollRecords", 500).toString()) + put(ConsumerConfig.FETCH_MIN_BYTES_CONFIG, config.int("consumer.fetchMinBytes", 1).toString()) + put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, config.int("consumer.fetchMaxWaitMs", 500).toString()) + put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, config.int("consumer.sessionTimeoutMs", 45_000).toString()) + put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, config.int("consumer.heartbeatIntervalMs", 15_000).toString()) + put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, config.int("consumer.maxPollIntervalMs", 300_000).toString()) + } +} diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataMeter.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataMeter.kt new file mode 100644 index 000000000..e9e51e391 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataMeter.kt @@ -0,0 +1,29 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.config + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Timer + +class RawDataMeter(registry: MeterRegistry) { + val deserializeFailures: Counter = registry.counter("rawdata_deserialize_failures") + val failures: Counter = registry.counter("rawdata_failures") + val enqueued: Counter = registry.counter("rawdata_enqueued") + val dequeued: Counter = registry.counter("rawdata_dequeued") + val saved: Counter = registry.counter("rawdata_saved") + val savingLatency: Timer = registry.timer("rawdata_saving_latency") +} \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataQueueConfig.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataQueueConfig.kt new file mode 100644 index 000000000..a04026399 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataQueueConfig.kt @@ -0,0 +1,61 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.config + +import io.ktor.server.config.ApplicationConfig + +/** + * Configuration for the raw data queue. + * + * @param config Application config scoped to `drill.rawData.queue`. + * @param kafkaConfig Shared Kafka cluster and connection configuration from `drill.kafka`. + */ +class RawDataQueueConfig( + private val config: ApplicationConfig, + private val kafkaConfig: KafkaConfig, +) { + /** + * Defines the raw data queue implementation. + */ + val type: RawDataQueueType + get() = config.propertyOrNull("type")?.getString()?.let { RawDataQueueType.valueOf(it) } ?: RawDataQueueType.IN_MEMORY + + /** + * Defines the capacity of the queue used for processing incoming raw data. + * If the queue reaches its capacity, processing of new data will be suspended until there is space available. + */ + val capacity: Int + get() = config.propertyOrNull("capacity")?.getString()?.toIntOrNull() ?: 1000 + + /** + * Defines the number of concurrent workers that will process the raw data from the queue. + * Should be less than database connection pool size. + */ + val workers: Int + get() = config.propertyOrNull("workers")?.getString()?.toIntOrNull() ?: 10 + + /** + * Kafka-specific queue configuration (topic, consumer group, poll/shutdown timeouts, + * producer/consumer tuning). Cluster connection settings are delegated to [kafkaConfig]. + */ + val kafka: RawDataKafkaQueueConfig + get() = RawDataKafkaQueueConfig(config.config("kafka"), kafkaConfig) +} + +enum class RawDataQueueType { + IN_MEMORY, + KAFKA +} diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt index dbea1407a..b155b6426 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt @@ -16,25 +16,41 @@ package com.epam.drill.admin.writer.rawdata.config import com.epam.drill.admin.writer.rawdata.job.DataRetentionPolicyJob +import com.epam.drill.admin.writer.rawdata.queue.DataQueue +import com.epam.drill.admin.writer.rawdata.queue.impl.ChannelDataQueue +import com.epam.drill.admin.writer.rawdata.queue.impl.KafkaDataQueue +import com.epam.drill.admin.writer.rawdata.service.QueuedRawDataWriter +import com.epam.drill.admin.writer.rawdata.queue.impl.json import com.epam.drill.admin.writer.rawdata.repository.* import com.epam.drill.admin.writer.rawdata.repository.impl.* +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload import com.epam.drill.admin.writer.rawdata.service.DataManagementService import com.epam.drill.admin.writer.rawdata.service.RawDataWriter import com.epam.drill.admin.writer.rawdata.service.SettingsService import com.epam.drill.admin.writer.rawdata.service.impl.DataManagementServiceImpl import com.epam.drill.admin.writer.rawdata.service.impl.RawDataServiceImpl import com.epam.drill.admin.writer.rawdata.service.impl.SettingsServiceImpl +import com.epam.drill.admin.writer.rawdata.service.impl.toKey +import com.epam.drill.admin.writer.rawdata.service.impl.toPayloadType +import io.ktor.server.application.Application +import io.ktor.server.config.ApplicationConfig import org.kodein.di.DI import org.kodein.di.bind +import org.kodein.di.eagerSingleton import org.kodein.di.instance import org.kodein.di.singleton import org.quartz.JobBuilder import org.quartz.JobDetail +import kotlin.time.Duration.Companion.milliseconds + +private val logger = mu.KotlinLogging.logger {} val rawDataDIModule get() = DI.Module("rawDataServices") { - import(rawDataServicesDIModule) + importOnce(rawDataServicesDIModule) importOnce(settingsServicesDIModule) + importOnce(dataManagementServicesDIModule) bind() with singleton { DataRetentionPolicyJob( @@ -51,6 +67,7 @@ val rawDataDIModule val rawDataServicesDIModule get() = DI.Module("rawDataWriterServices") { + bind() with singleton { RawDataMeter(instance()) } bind() with singleton { InstanceRepositoryImpl() } bind() with singleton { BuildRepositoryImpl() } bind() with singleton { MethodRepositoryImpl() } @@ -59,7 +76,6 @@ val rawDataServicesDIModule bind() with singleton { TestSessionRepositoryImpl() } bind() with singleton { TestSessionBuildRepositoryImpl() } bind() with singleton { TestLaunchRepositoryImpl() } - bind() with singleton { MethodIgnoreRuleRepositoryImpl() } bind() with singleton { RawDataServiceImpl( @@ -71,21 +87,65 @@ val rawDataServicesDIModule buildRepository = instance(), testSessionRepository = instance(), testSessionBuildRepository = instance(), - methodIgnoreRuleRepository = instance() ) } - bind() with singleton { - DataManagementServiceImpl( - instanceRepository = instance(), - buildRepository = instance(), - methodRepository = instance(), - coverageRepository = instance(), - testSessionRepository = instance(), - testLaunchRepository = instance(), - testSessionBuildRepository = instance(), - scheduler = instance(), + + bind() with singleton { + val env = instance().environment.config + KafkaConfig(env.config("kafka")) + } + bind() with singleton { + val drillConfig: ApplicationConfig = instance().environment.config.config("drill") + RawDataQueueConfig(drillConfig.config("rawData.queue"), instance()) + } + bind>(tag = RawDataQueueType.IN_MEMORY) with singleton { + val config = instance() + ChannelDataQueue( + deserializer = ::json, + routeToPayloadType = { route -> + route.toPayloadType() + }, + capacity = config.capacity, + metrics = instance(), + ) + } + bind>(tag = RawDataQueueType.KAFKA) with singleton { + val config = instance() + val kafkaQueueConfig = config.kafka + val kafkaClusterConfig = instance() + KafkaDataQueue.create( + bootstrapServers = kafkaClusterConfig.bootstrapServers, + topic = kafkaQueueConfig.topic, + consumerGroupId = kafkaQueueConfig.consumerGroupId, + deserializer = ::json, + recordKeyToPayloadType = { key -> + key.toPayloadType() + }, + routeToRecordKey = { route -> + route.toKey() + }, + producerProps = kafkaQueueConfig.producerProperties, + consumerProps = kafkaQueueConfig.consumerProperties, + capacity = config.capacity, + pollTimeout = kafkaQueueConfig.pollTimeoutMs.milliseconds, + shutdownTimeout = kafkaQueueConfig.shutdownTimeoutMs.milliseconds, + metrics = instance(), ) } + bind() with eagerSingleton { + val config = instance() + val writer = instance() + val queue = instance>(tag = config.type) + + QueuedRawDataWriter( + handler = writer, + workers = config.workers, + queue = queue, + metrics = instance(), + ).also { + logger.info { "${config.type} queue is configured for raw data writing with ${config.workers} workers." } + } + } } val settingsServicesDIModule @@ -100,6 +160,24 @@ val settingsServicesDIModule } } +val dataManagementServicesDIModule + get() = DI.Module("dataManagementServices") { + bind() with singleton { MethodIgnoreRuleRepositoryImpl() } + bind() with singleton { + DataManagementServiceImpl( + instanceRepository = instance(), + buildRepository = instance(), + methodRepository = instance(), + coverageRepository = instance(), + testSessionRepository = instance(), + testLaunchRepository = instance(), + testSessionBuildRepository = instance(), + methodIgnoreRuleRepository = instance(), + scheduler = instance(), + ) + } + } + val rawDataRetentionPolicyJob: JobDetail get() = JobBuilder.newJob(DataRetentionPolicyJob::class.java) .storeDurably() diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/DataQueue.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/DataQueue.kt new file mode 100644 index 000000000..d6109044b --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/DataQueue.kt @@ -0,0 +1,36 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue + +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlinx.coroutines.channels.ReceiveChannel + +interface DataQueue : ReceiveChannel> { + suspend fun enqueue(input: QueueInput) + suspend fun dequeue(): QueueOutput +} + +class QueueInput( + val route: R, + val payload: ByteArray, + val metadata: Map = emptyMap() +) + +class QueueOutput( + val payload: T, + val metadata: Map = emptyMap() +) \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/QueueProcessor.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/QueueProcessor.kt new file mode 100644 index 000000000..7391e843f --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/QueueProcessor.kt @@ -0,0 +1,47 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue + +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch + +class QueueProcessor( + private val handler: suspend (QueueOutput) -> Unit, + private val onError: suspend (QueueOutput, Throwable) -> Unit = { _, _ -> }, + private val onSuccess: suspend (QueueOutput) -> Unit = {}, +) { + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + fun run(queue: DataQueue, workers: Int = 1) { + repeat(workers) { worker -> + scope.launch { + for (output in queue) { + runCatching { + handler(output) + }.onFailure { e -> + onError(output, e) + }.onSuccess { + onSuccess(output) + } + } + } + } + } +} diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueue.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueue.kt new file mode 100644 index 000000000..3e1804469 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueue.kt @@ -0,0 +1,85 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.impl + +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import com.epam.drill.admin.writer.rawdata.queue.DataQueue +import com.epam.drill.admin.writer.rawdata.queue.QueueInput +import com.epam.drill.admin.writer.rawdata.queue.QueueOutput +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.time.withTimeout +import mu.KotlinLogging +import kotlin.reflect.KClass +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration + +class ChannelDataQueue( + private val deserializer: suspend (KClass, ByteArray) -> T, + private val routeToPayloadType: (R) -> KClass, + capacity: Int = Channel.BUFFERED, + private val shutdownTimeout: Duration = 5.seconds, + private val metrics: RawDataMeter, +) : DataQueue, Channel> by Channel(capacity), AutoCloseable { + private val logger = KotlinLogging.logger {} + private val inputChannel = Channel>(Channel.RENDEZVOUS) + private val outputChannel: Channel> get() = this + private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + + init { + scope.launch { + for (input in inputChannel) { + val payloadType = routeToPayloadType(input.route) + runCatching { + deserializer(payloadType, input.payload) + }.onFailure { e -> + logger.error(e) { "Error while deserialization queue for [${input.route::class.simpleName}]: ${e.message}" } + metrics.deserializeFailures.increment() + }.getOrNull()?.let { payload -> + outputChannel.send(QueueOutput(payload, input.metadata)) + } ?: continue + } + } + } + + override suspend fun enqueue(input: QueueInput) { + inputChannel.send(input) + } + + override suspend fun dequeue(): QueueOutput { + return outputChannel.receive() + } + + override fun close() { + inputChannel.close() + outputChannel.close() + runBlocking { + withTimeout(shutdownTimeout.toJavaDuration()) { + scope.coroutineContext[Job]?.children?.forEach { it.join() } + } + scope.cancel() + } + } +} \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/JsonDeserializer.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/JsonDeserializer.kt new file mode 100644 index 000000000..31e286352 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/JsonDeserializer.kt @@ -0,0 +1,36 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.impl + +import com.epam.drill.admin.writer.rawdata.route.jsonConfig +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlinx.serialization.KSerializer +import kotlinx.serialization.json.Json +import kotlinx.serialization.serializer +import kotlin.reflect.KClass + +class JsonDeserializer( + private val serializer: KSerializer, + private val json: Json = jsonConfig +) { + fun deserialize(bytes: ByteArray): T { + val decoded = bytes.toString(Charsets.UTF_8) + return json.decodeFromString(serializer, decoded) + } +} + +fun json(type: KClass, bytes: ByteArray): T = + JsonDeserializer(type.serializer()).deserialize(bytes) \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueue.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueue.kt new file mode 100644 index 000000000..a5ba6e7bf --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueue.kt @@ -0,0 +1,228 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.impl + +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import com.epam.drill.admin.writer.rawdata.queue.DataQueue +import com.epam.drill.admin.writer.rawdata.queue.QueueInput +import com.epam.drill.admin.writer.rawdata.queue.QueueOutput +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.SendChannel +import kotlinx.coroutines.future.await +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.time.withTimeout +import mu.KotlinLogging +import org.apache.kafka.clients.consumer.Consumer +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.Producer +import org.apache.kafka.clients.producer.ProducerConfig +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.errors.WakeupException +import org.apache.kafka.common.header.internals.RecordHeader +import org.apache.kafka.common.serialization.ByteArrayDeserializer +import org.apache.kafka.common.serialization.ByteArraySerializer +import org.apache.kafka.common.serialization.StringDeserializer +import org.apache.kafka.common.serialization.StringSerializer +import java.util.Properties +import java.util.concurrent.CompletableFuture +import kotlin.coroutines.coroutineContext +import kotlin.reflect.KClass +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration +import java.time.Duration as JavaDuration + +/** + * Kafka-backed implementation of [DataQueue]. + * + * @param producer pre-configured Kafka [Producer]; closed by [close]. + * @param consumer pre-configured Kafka [Consumer]; subscribed to [topic] and closed by [close]. + * @param topic single Kafka topic used for both publishing and consuming raw payloads. + * @param deserializer converts a route + payload bytes into a typed [RawDataPayload]. + * @param recordKeyToPayloadType maps the route key (extracted from the record header) back to the corresponding payload type for deserialization. + * @param routeToRecordKey extracts the key for a given route instance (defaults to its class simple name). + * @param RECORD_KEY_HEADER Kafka header name used to carry the route key. + * @param capacity capacity of the internal channel exposed to consumers of this queue. + * @param pollTimeout per-poll timeout used by the consumer loop. + * @param shutdownTimeout maximum time to wait for the background coroutines to finish on [close]. + */ +class KafkaDataQueue( + private val producer: Producer, + private val consumer: Consumer, + private val topic: String, + private val deserializer: suspend (KClass, ByteArray) -> T, + private val recordKeyToPayloadType: (String) -> KClass, + private val routeToRecordKey: (R) -> String, + capacity: Int = Channel.RENDEZVOUS, + private val pollTimeout: Duration = 500.milliseconds, + private val shutdownTimeout: Duration = 5.seconds, + private val metrics: RawDataMeter, +) : DataQueue, Channel> by Channel(capacity), AutoCloseable { + + private val logger = KotlinLogging.logger {} + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val outputChannel: SendChannel> get() = this + + init { + consumer.subscribe(listOf(topic)) + scope.launch { runConsumerLoop() } + } + + override suspend fun enqueue(input: QueueInput) { + val recordKey = routeToRecordKey(input.route) + val record = ProducerRecord(topic, null, recordKey, input.payload).apply { + headers().add(RecordHeader(RECORD_KEY_HEADER, recordKey.toByteArray(Charsets.UTF_8))) + input.metadata.forEach { (k, v) -> + headers().add(RecordHeader(k, v.toByteArray(Charsets.UTF_8))) + } + } + // Bridge the Kafka producer's java callback to a coroutine. + val future = CompletableFuture() + producer.send(record) { _, e -> + if (e != null) future.completeExceptionally(e) else future.complete(Unit) + } + future.await() + } + + override suspend fun dequeue(): QueueOutput = this.receive() + + override fun close() { + runCatching { consumer.wakeup() } + runBlocking { + withTimeout(shutdownTimeout.toJavaDuration()) { + scope.coroutineContext[Job]?.children?.forEach { it.join() } + } + scope.cancel() + } + runCatching { consumer.close(shutdownTimeout.toJavaDuration()) } + runCatching { producer.close(shutdownTimeout.toJavaDuration()) } + outputChannel.close() + } + + private suspend fun runConsumerLoop() { + val pollDuration: JavaDuration = pollTimeout.toJavaDuration() + try { + while (coroutineContext[Job]?.isActive == true) { + val records = try { + consumer.poll(pollDuration) + } catch (e: WakeupException) { + throw e + } catch (e: Throwable) { + logger.error(e) { "Error while polling Kafka topic [$topic]: ${e.message}" } + null + } ?: continue + + for (record in records) { + val key = record.key() + if (key == null) { + logger.warn { "Skipping Kafka record without record key on topic [$topic]" } + continue + } + + val payloadType = runCatching { + recordKeyToPayloadType(key) + }.onFailure { e -> + logger.error(e) { "Error while determining payload type for [$key]: ${e.message}" } + }.getOrNull() ?: continue + + val payload = runCatching { + deserializer(payloadType, record.value()) + }.onFailure { e -> + logger.error(e) { "Error while deserializing record for [$key]: ${e.message}" } + metrics.deserializeFailures.increment() + }.getOrNull() ?: continue + + val metadata = record.headers() + .associate { it.key() to it.value().toString(Charsets.UTF_8) } + + outputChannel.send(QueueOutput(payload, metadata)) + } + + runCatching { + consumer.commitAsync() + }.onFailure { e -> + logger.warn(e) { "Kafka commitAsync failed: ${e.message}" } + } + } + } catch (_: WakeupException) { + logger.debug { "Kafka consumer woken up, exiting poll loop" } + } catch (e: Throwable) { + logger.error(e) { "Kafka consumer loop terminated unexpectedly: ${e.message}" } + } + } + + companion object { + const val RECORD_KEY_HEADER = "drill-record-key" + + /** + * Convenience factory that builds Kafka producer/consumer from raw [Properties]. + * Key/value (de)serializers are forced to String/ByteArray. + */ + fun create( + bootstrapServers: String, + topic: String, + consumerGroupId: String, + deserializer: suspend (KClass, ByteArray) -> T, + recordKeyToPayloadType: (String) -> KClass, + routeToRecordKey: (R) -> String, + producerProps: Properties = Properties(), + consumerProps: Properties = Properties(), + capacity: Int = Channel.BUFFERED, + pollTimeout: Duration = 500.milliseconds, + shutdownTimeout: Duration = 5.seconds, + metrics: RawDataMeter, + ): KafkaDataQueue { + val pProps = Properties().apply { + putAll(producerProps) + put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer::class.java.name) + put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer::class.java.name) + } + val cProps = Properties().apply { + putAll(consumerProps) + put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId) + put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.java.name) + put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer::class.java.name) + putIfAbsent(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false") + putIfAbsent(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") + } + return KafkaDataQueue( + producer = KafkaProducer(pProps), + consumer = KafkaConsumer(cProps), + topic = topic, + deserializer = deserializer, + recordKeyToPayloadType = recordKeyToPayloadType, + routeToRecordKey = routeToRecordKey, + capacity = capacity, + pollTimeout = pollTimeout, + shutdownTimeout = shutdownTimeout, + metrics = metrics, + ) + } + } +} \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/record/RecordKey.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/record/RecordKey.kt new file mode 100644 index 000000000..3f8d81501 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/queue/record/RecordKey.kt @@ -0,0 +1,45 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.record + +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestDefinitionsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestLaunchesPayload +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.BuildInfoPayload +import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload +import com.epam.drill.admin.writer.rawdata.route.payload.CoveragePayload +import com.epam.drill.admin.writer.rawdata.route.payload.InstancePayload +import com.epam.drill.admin.writer.rawdata.route.payload.MethodsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import com.epam.drill.admin.writer.rawdata.route.payload.SessionPayload +import kotlin.reflect.KClass + +enum class RecordKey(val value: String, val payloadType: KClass) { + COVERAGE("coverage", CoveragePayload::class), + BUILDS_INFO("builds-info", BuildInfoPayload::class), + BUILDS("builds", BuildPayload::class), + INSTANCES("instances", InstancePayload::class), + METHODS("methods", MethodsPayload::class), + TEST_DEFINITIONS("test-definitions", AddTestDefinitionsPayload::class), + TEST_LAUNCHES("test-launches", AddTestLaunchesPayload::class), + TEST_METADATA("test-metadata", AddTestsPayload::class), + TEST_SESSIONS("test-sessions", SessionPayload::class); + + companion object { + private val map = RecordKey.entries.associateBy(RecordKey::value) + fun fromValue(value: String): RecordKey = map[value] ?: throw IllegalArgumentException("Unknown RecordKey value: $value") + } +} \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt index 09b3bf4ad..6d024f609 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt @@ -17,7 +17,8 @@ package com.epam.drill.admin.writer.rawdata.route import com.epam.drill.admin.common.principal.User import com.epam.drill.admin.common.route.ok -import com.epam.drill.admin.writer.rawdata.service.RawDataWriter +import com.epam.drill.admin.writer.rawdata.service.QueuedRawDataWriter +import com.epam.drill.admin.writer.rawdata.service.DataManagementService import io.ktor.client.* import io.ktor.client.engine.apache.* import io.ktor.client.plugins.contentnegotiation.* @@ -43,35 +44,38 @@ import org.kodein.di.instance import org.kodein.di.ktor.closestDI import java.io.InputStream import java.util.zip.GZIPInputStream +import kotlin.getValue private val logger = KotlinLogging.logger {} +sealed interface DataIngestRoute + @Resource("builds") -class BuildsRoute() +class BuildsRoute(): DataIngestRoute @Resource("builds/info") -class BuildsInfoRoute() +class BuildsInfoRoute(): DataIngestRoute @Resource("instances") -class InstancesRoute() +class InstancesRoute(): DataIngestRoute @Resource("coverage") -class CoverageRoute() +class CoverageRoute(): DataIngestRoute @Resource("methods") -class MethodsRoute() +class MethodsRoute(): DataIngestRoute @Resource("tests-metadata") -class TestMetadataRoute() +class TestMetadataRoute(): DataIngestRoute @Resource("sessions") -class TestSessionRoute() +class TestSessionRoute(): DataIngestRoute @Resource("test-definitions") -class TestDefinitionsRoute() +class TestDefinitionsRoute(): DataIngestRoute @Resource("test-launches") -class TestLaunchesRoute() +class TestLaunchesRoute(): DataIngestRoute @Resource("method-ignore-rules") class MethodIgnoreRulesRoute() { @@ -100,109 +104,109 @@ fun Route.dataIngestRoutes() { } fun Route.putBuilds() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - put { - rawDataWriter.saveBuild(call.decompressAndReceive()) + put { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Build saved") } } fun Route.putBuildsInfo() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - put { - rawDataWriter.saveBuildInfo(call.decompressAndReceive()) + put { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Build info saved") } } fun Route.putInstances() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - put { - rawDataWriter.saveInstance(call.decompressAndReceive()) + put { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Instance saved") } } fun Route.postCoverage() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - post { - rawDataWriter.saveCoverage(call.decompressAndReceive()) + post { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Coverage saved") } } fun Route.putMethods() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - put { - rawDataWriter.saveMethods(call.decompressAndReceive()) + put { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Methods saved") } } fun Route.postTestMetadata() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - post { - rawDataWriter.saveTestMetadata(call.decompressAndReceive()) + post { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Test metadata saved") } } fun Route.putTestSessions() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - put { - rawDataWriter.saveTestSession(call.decompressAndReceive(), call.principal()) + put { params -> + queuedRawDataWriter.enqueue(params, call.decompress(), call.principal()?.username) call.ok("Test sessions saved") } } fun Route.postTestDefinitions() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - post { - rawDataWriter.saveTestDefinitions(call.decompressAndReceive()) + post { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Test definitions saved") } } fun Route.postTestLaunches() { - val rawDataWriter by closestDI().instance() + val queuedRawDataWriter by closestDI().instance() - post { - rawDataWriter.saveTestLaunches(call.decompressAndReceive()) + post { params -> + queuedRawDataWriter.enqueue(params, call.decompress()) call.ok("Test launches saved") } } fun Route.postMethodIgnoreRules() { - val rawDataWriter by closestDI().instance() + val dataManagementService by closestDI().instance() post { - rawDataWriter.saveMethodIgnoreRule(call.decompressAndReceive()) + dataManagementService.saveMethodIgnoreRule(call.decompressAndReceive()) call.ok("Method ignore rule saved") } } fun Route.getMethodIgnoreRules() { - val rawDataWriter by closestDI().instance() + val dataManagementService by closestDI().instance() get { - call.ok(rawDataWriter.getAllMethodIgnoreRules()) + call.ok(dataManagementService.getAllMethodIgnoreRules()) } } fun Route.deleteMethodIgnoreRule() { - val rawDataWriter by closestDI().instance() + val dataManagementService by closestDI().instance() delete { params -> val id = params.id - rawDataWriter.deleteMethodIgnoreRuleById(id) + dataManagementService.deleteMethodIgnoreRuleById(id) call.ok("Method ignore rule deleted") } } @@ -240,32 +244,37 @@ internal suspend fun sendPostRequest(url: String, data: Any) { } } -internal val json = Json { +internal val jsonConfig = Json { ignoreUnknownKeys = true explicitNulls = false } -/** - * Workaround for decompressing the request body before upgrading to Ktor 3.0.0, where this feature works out of the box - * https://github.com/ktorio/ktor/issues/3845 - */ internal suspend inline fun ApplicationCall.decompressAndReceive(): T { - val body: ByteArray = when (request.headers[HttpHeaders.ContentEncoding]) { - "gzip" -> decompressGZip(receiveStream()) - else -> receive() - } + val body: ByteArray = decompress() return when (request.headers[HttpHeaders.ContentType]) { ContentType.Application.ProtoBuf.toString() -> ProtoBuf.decodeFromByteArray(T::class.serializer(), body) - ContentType.Application.Json.toString() -> json.decodeFromString(T::class.serializer(), String(body)) + ContentType.Application.Json.toString() -> jsonConfig.decodeFromString(T::class.serializer(), String(body)) else -> throw request.headers[HttpHeaders.ContentType]?.let { UnsupportedMediaTypeException(ContentType.parse(it)) } ?: BadRequestException("Content-Type header is missing") } } + internal suspend fun decompressGZip(inputStream: InputStream): ByteArray { val decompressedBytes = withContext(Dispatchers.IO) { GZIPInputStream(inputStream).readBytes() } return decompressedBytes } + +/** + * TODO: Workaround for decompressing the request body before upgrading to Ktor 3.0.0, where this feature works out of the box + * https://github.com/ktorio/ktor/issues/3845 + */ +internal suspend inline fun ApplicationCall.decompress(): ByteArray { + return when (request.headers[HttpHeaders.ContentEncoding]) { + "gzip" -> decompressGZip(receiveStream()) + else -> receive() + } +} diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildInfoPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildInfoPayload.kt new file mode 100644 index 000000000..506f04c49 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildInfoPayload.kt @@ -0,0 +1,30 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.route.payload + +import kotlinx.serialization.Serializable + +@Serializable +class BuildInfoPayload( + val groupId: String, + val appId: String, + val commitSha: String? = null, + val buildVersion: String? = null, + val branch: String? = null, + val commitDate: String? = null, + val commitMessage: String? = null, + val commitAuthor: String? = null +): RawDataPayload \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildPayload.kt index 4cb70ec3f..895422ec2 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildPayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/BuildPayload.kt @@ -15,7 +15,6 @@ */ package com.epam.drill.admin.writer.rawdata.route.payload -import kotlinx.datetime.Instant import kotlinx.serialization.Serializable @Serializable @@ -24,8 +23,4 @@ class BuildPayload( val appId: String, val commitSha: String? = null, val buildVersion: String? = null, - val branch: String? = null, - val commitDate: String? = null, - val commitMessage: String? = null, - val commitAuthor: String? = null -) +): RawDataPayload diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/CoveragePayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/CoveragePayload.kt index 6d9115b28..227dc8215 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/CoveragePayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/CoveragePayload.kt @@ -25,7 +25,7 @@ class CoveragePayload( val commitSha: String?, val buildVersion: String?, val coverage: List -) +): RawDataPayload @Serializable class SingleMethodCoveragePayload( diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/InstancePayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/InstancePayload.kt index f508e1e75..ccffe8949 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/InstancePayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/InstancePayload.kt @@ -25,4 +25,4 @@ class InstancePayload( val commitSha: String? = null, val buildVersion: String? = null, val envId: String? = null, -) \ No newline at end of file +): RawDataPayload \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/MethodPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/MethodPayload.kt index 44318c7bb..ddfc18236 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/MethodPayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/MethodPayload.kt @@ -25,7 +25,7 @@ class MethodsPayload( val buildVersion: String? = null, val instanceId: String? = null, val methods: Array -) +) : RawDataPayload @Serializable class SingleMethodPayload( diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/RawDataPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/RawDataPayload.kt new file mode 100644 index 000000000..dbcc9e374 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/RawDataPayload.kt @@ -0,0 +1,18 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.route.payload + +sealed interface RawDataPayload \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt index b124f5caa..d6cc3b2e8 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt @@ -25,7 +25,7 @@ class AddTestsPayload( val groupId: String, val sessionId: String, val tests: List = emptyList(), -) +): RawDataPayload @Serializable class TestLaunchInfo( @@ -70,14 +70,14 @@ class SessionPayload( val testTaskId: String, val startedAt: Instant, val builds: List = emptyList(), -) +): RawDataPayload @Serializable class AddTestLaunchesPayload( val groupId: String, val testSessionId: String, val launches: List, -) +): RawDataPayload @Serializable class TestLaunchPayload ( @@ -92,7 +92,7 @@ class TestLaunchPayload ( class AddTestDefinitionsPayload( val groupId: String, val definitions: List -) +): RawDataPayload // TODO: update test agent // Order of fields, and field definitions changed compared to original TestDefinition class: diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/DataManagementService.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/DataManagementService.kt index 60999b83b..53677f6dd 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/DataManagementService.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/DataManagementService.kt @@ -16,6 +16,8 @@ package com.epam.drill.admin.writer.rawdata.service import com.epam.drill.admin.common.principal.User +import com.epam.drill.admin.writer.rawdata.route.payload.MethodIgnoreRulePayload +import com.epam.drill.admin.writer.rawdata.views.MethodIgnoreRuleView interface DataManagementService { /** @@ -33,4 +35,8 @@ interface DataManagementService { * @param user The user performing the deletion (optional). */ suspend fun deleteTestSessionData(groupId: String, testSessionId: String, user: User?) + + suspend fun saveMethodIgnoreRule(rulePayload: MethodIgnoreRulePayload) + suspend fun getAllMethodIgnoreRules(): List + suspend fun deleteMethodIgnoreRuleById(ruleId: Int) } \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/QueuedRawDataWriter.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/QueuedRawDataWriter.kt new file mode 100644 index 000000000..59e3ed087 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/QueuedRawDataWriter.kt @@ -0,0 +1,82 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.service + +import com.epam.drill.admin.common.config.recordSuspend +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import com.epam.drill.admin.writer.rawdata.queue.DataQueue +import com.epam.drill.admin.writer.rawdata.queue.QueueInput +import com.epam.drill.admin.writer.rawdata.queue.QueueProcessor +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestDefinitionsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestLaunchesPayload +import com.epam.drill.admin.writer.rawdata.route.payload.AddTestsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.BuildInfoPayload +import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload +import com.epam.drill.admin.writer.rawdata.route.payload.CoveragePayload +import com.epam.drill.admin.writer.rawdata.route.payload.InstancePayload +import com.epam.drill.admin.writer.rawdata.route.payload.MethodsPayload +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import com.epam.drill.admin.writer.rawdata.route.payload.SessionPayload +import mu.KotlinLogging + +class QueuedRawDataWriter( + handler: RawDataWriter, + workers: Int = 10, + private val queue: DataQueue, + private val metrics: RawDataMeter, +) { + private val logger = KotlinLogging.logger {} + private val usernameKey = "username" + private val queueProcessor = QueueProcessor( + handler = { output -> + metrics.dequeued.increment() + metrics.savingLatency.recordSuspend { + val payload = output.payload + val metadata = output.metadata + when (payload) { + is CoveragePayload -> handler.saveCoverage(payload) + is MethodsPayload -> handler.saveMethods(payload) + is BuildPayload -> handler.saveBuild(payload) + is BuildInfoPayload -> handler.saveBuildInfo(payload) + is AddTestDefinitionsPayload -> handler.saveTestDefinitions(payload) + is AddTestLaunchesPayload -> handler.saveTestLaunches(payload) + is AddTestsPayload -> handler.saveTestMetadata(payload) + is InstancePayload -> handler.saveInstance(payload) + is SessionPayload -> handler.saveTestSession(payload, metadata[usernameKey]) + } + } + }, + onError = { output, e -> + logger.error(e) { "Error while saving [${output.payload::class.simpleName}]: ${e.message}" } + metrics.failures.increment() + }, + onSuccess = { output -> + logger.debug { "Successfully saved [${output.payload::class.simpleName}]" } + metrics.saved.increment() + } + ) + + init { + queueProcessor.run(queue, workers) + } + + suspend fun enqueue(route: DataIngestRoute, payload: ByteArray, username: String? = null) { + val metadata = username?.let { mapOf(usernameKey to it) } ?: emptyMap() + queue.enqueue(QueueInput(route, payload, metadata)) + metrics.enqueued.increment() + } +} \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt index ebdca25d3..dc2e44f19 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt @@ -21,15 +21,12 @@ import com.epam.drill.admin.writer.rawdata.views.MethodIgnoreRuleView interface RawDataWriter { suspend fun saveBuild(buildPayload: BuildPayload) - suspend fun saveBuildInfo(buildPayload: BuildPayload) + suspend fun saveBuildInfo(buildPayload: BuildInfoPayload) suspend fun saveInstance(instancePayload: InstancePayload) suspend fun saveMethods(methodsPayload: MethodsPayload) suspend fun saveCoverage(coveragePayload: CoveragePayload) suspend fun saveTestMetadata(testsPayload: AddTestsPayload) suspend fun saveTestDefinitions(testDefinitionsPayload: AddTestDefinitionsPayload) suspend fun saveTestLaunches(testLaunchesPayload: AddTestLaunchesPayload) - suspend fun saveTestSession(sessionPayload: SessionPayload, user: User?) - suspend fun saveMethodIgnoreRule(rulePayload: MethodIgnoreRulePayload) - suspend fun getAllMethodIgnoreRules(): List - suspend fun deleteMethodIgnoreRuleById(ruleId: Int) + suspend fun saveTestSession(sessionPayload: SessionPayload, username: String?) } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/DataManagementServiceImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/DataManagementServiceImpl.kt index 14efde7e8..25006afea 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/DataManagementServiceImpl.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/DataManagementServiceImpl.kt @@ -22,14 +22,18 @@ import com.epam.drill.admin.common.scheduler.deleteMetricsDataJobKey import com.epam.drill.admin.common.scheduler.getBuildDataDeletionDataMap import com.epam.drill.admin.common.scheduler.getTestDataDeletionDataMap import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig.transaction +import com.epam.drill.admin.writer.rawdata.entity.MethodIgnoreRule import com.epam.drill.admin.writer.rawdata.repository.BuildRepository import com.epam.drill.admin.writer.rawdata.repository.CoverageRepository import com.epam.drill.admin.writer.rawdata.repository.InstanceRepository +import com.epam.drill.admin.writer.rawdata.repository.MethodIgnoreRuleRepository import com.epam.drill.admin.writer.rawdata.repository.MethodRepository import com.epam.drill.admin.writer.rawdata.repository.TestLaunchRepository import com.epam.drill.admin.writer.rawdata.repository.TestSessionBuildRepository import com.epam.drill.admin.writer.rawdata.repository.TestSessionRepository +import com.epam.drill.admin.writer.rawdata.route.payload.MethodIgnoreRulePayload import com.epam.drill.admin.writer.rawdata.service.DataManagementService +import com.epam.drill.admin.writer.rawdata.views.MethodIgnoreRuleView class DataManagementServiceImpl( private val buildRepository: BuildRepository, @@ -39,6 +43,7 @@ class DataManagementServiceImpl( private val methodRepository: MethodRepository, private val testSessionBuildRepository: TestSessionBuildRepository, private val testLaunchRepository: TestLaunchRepository, + private val methodIgnoreRuleRepository: MethodIgnoreRuleRepository, private val scheduler: DrillScheduler, ) : DataManagementService { @@ -68,5 +73,29 @@ class DataManagementServiceImpl( scheduler.triggerJob(deleteMetricsDataJobKey, getTestDataDeletionDataMap(groupId, testSessionId)) } } + + override suspend fun saveMethodIgnoreRule(rulePayload: MethodIgnoreRulePayload) { + val rule = MethodIgnoreRule( + groupId = rulePayload.groupId, + appId = rulePayload.appId, + namePattern = rulePayload.namePattern, + classnamePattern = rulePayload.classnamePattern, + ) + transaction { + methodIgnoreRuleRepository.create(rule) + } + } + + override suspend fun getAllMethodIgnoreRules(): List { + return transaction { + methodIgnoreRuleRepository.getAll() + } + } + + override suspend fun deleteMethodIgnoreRuleById(ruleId: Int) { + transaction { + methodIgnoreRuleRepository.deleteById(ruleId) + } + } } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataMappers.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataMappers.kt new file mode 100644 index 000000000..01e1177e5 --- /dev/null +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataMappers.kt @@ -0,0 +1,48 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.service.impl + +import com.epam.drill.admin.writer.rawdata.queue.record.RecordKey +import com.epam.drill.admin.writer.rawdata.route.BuildsInfoRoute +import com.epam.drill.admin.writer.rawdata.route.BuildsRoute +import com.epam.drill.admin.writer.rawdata.route.CoverageRoute +import com.epam.drill.admin.writer.rawdata.route.DataIngestRoute +import com.epam.drill.admin.writer.rawdata.route.InstancesRoute +import com.epam.drill.admin.writer.rawdata.route.MethodsRoute +import com.epam.drill.admin.writer.rawdata.route.TestDefinitionsRoute +import com.epam.drill.admin.writer.rawdata.route.TestLaunchesRoute +import com.epam.drill.admin.writer.rawdata.route.TestMetadataRoute +import com.epam.drill.admin.writer.rawdata.route.TestSessionRoute +import com.epam.drill.admin.writer.rawdata.route.payload.RawDataPayload +import kotlin.reflect.KClass + +fun DataIngestRoute.toRecordKey(): RecordKey = when (this) { + is CoverageRoute -> RecordKey.COVERAGE + is BuildsInfoRoute -> RecordKey.BUILDS_INFO + is BuildsRoute -> RecordKey.BUILDS + is InstancesRoute -> RecordKey.INSTANCES + is MethodsRoute -> RecordKey.METHODS + is TestDefinitionsRoute -> RecordKey.TEST_DEFINITIONS + is TestLaunchesRoute -> RecordKey.TEST_LAUNCHES + is TestMetadataRoute -> RecordKey.TEST_METADATA + is TestSessionRoute -> RecordKey.TEST_SESSIONS +} + +fun DataIngestRoute.toPayloadType(): KClass = toRecordKey().payloadType + +fun DataIngestRoute.toKey(): String = toRecordKey().value + +fun String.toPayloadType(): KClass = RecordKey.fromValue(this).payloadType diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt index 9c227c91e..eae313a6a 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt @@ -15,7 +15,6 @@ */ package com.epam.drill.admin.writer.rawdata.service.impl -import com.epam.drill.admin.common.principal.User import com.epam.drill.admin.common.service.generateBuildId import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig.transaction import com.epam.drill.admin.writer.rawdata.config.toBitString @@ -24,7 +23,6 @@ import com.epam.drill.admin.writer.rawdata.repository.* import com.epam.drill.admin.writer.rawdata.route.payload.* import com.epam.drill.admin.writer.rawdata.service.RawDataWriter import com.epam.drill.admin.writer.rawdata.util.md5 -import com.epam.drill.admin.writer.rawdata.views.MethodIgnoreRuleView import kotlinx.datetime.TimeZone import kotlinx.datetime.toJavaLocalDateTime import kotlinx.datetime.toLocalDateTime @@ -44,7 +42,6 @@ class RawDataServiceImpl( private val buildRepository: BuildRepository, private val testSessionRepository: TestSessionRepository, private val testSessionBuildRepository: TestSessionBuildRepository, - private val methodIgnoreRuleRepository: MethodIgnoreRuleRepository ) : RawDataWriter { override suspend fun saveBuild(buildPayload: BuildPayload) { @@ -67,7 +64,7 @@ class RawDataServiceImpl( } } - override suspend fun saveBuildInfo(buildPayload: BuildPayload) { + override suspend fun saveBuildInfo(buildPayload: BuildInfoPayload) { val build = Build( id = generateBuildId( buildPayload.groupId, @@ -255,13 +252,13 @@ class RawDataServiceImpl( }.let { testLaunchRepository.createMany(it) } } - override suspend fun saveTestSession(sessionPayload: SessionPayload, user: User?) { + override suspend fun saveTestSession(sessionPayload: SessionPayload, username: String?) { val testSession = TestSession( id = sessionPayload.id, groupId = sessionPayload.groupId, testTaskId = sessionPayload.testTaskId, startedAt = sessionPayload.startedAt.toLocalDateTime(TimeZone.UTC).toJavaLocalDateTime(), - createdBy = user?.username + createdBy = username ) transaction { testSessionRepository.create(testSession) @@ -279,30 +276,6 @@ class RawDataServiceImpl( } } - override suspend fun saveMethodIgnoreRule(rulePayload: MethodIgnoreRulePayload) { - val rule = MethodIgnoreRule( - groupId = rulePayload.groupId, - appId = rulePayload.appId, - namePattern = rulePayload.namePattern, - classnamePattern = rulePayload.classnamePattern, - ) - transaction { - methodIgnoreRuleRepository.create(rule) - } - } - - override suspend fun getAllMethodIgnoreRules(): List { - return transaction { - methodIgnoreRuleRepository.getAll() - } - } - - override suspend fun deleteMethodIgnoreRuleById(ruleId: Int) { - transaction { - methodIgnoreRuleRepository.deleteById(ruleId) - } - } - private fun convertGitDefaultDateTime(commitDate: String): LocalDateTime { return ZonedDateTime.parse(commitDate, DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy Z", Locale.ENGLISH)) .toLocalDateTime() diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/BuildsApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/BuildsApiTest.kt index 7260a0a54..fa18e303e 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/BuildsApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/BuildsApiTest.kt @@ -67,18 +67,20 @@ class BuildsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) { ) } - val savedBuilds = BuildTable.selectAll() - .filter { it[BuildTable.groupId] == testGroup } - .filter { it[BuildTable.appId] == testApp } - .filter { it[BuildTable.buildVersion] == testBuildVersion } - assertEquals(1, savedBuilds.size) - savedBuilds.forEach { - assertNull(it[BuildTable.branch]) - assertNotNull(it[BuildTable.commitSha]) - assertNull(it[BuildTable.commitAuthor]) - assertNull(it[BuildTable.commitMessage]) - assertNull(it[BuildTable.committedAt]) - assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedBuilds = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testGroup } + .filter { it[BuildTable.appId] == testApp } + .filter { it[BuildTable.buildVersion] == testBuildVersion } + assertEquals(1, savedBuilds.size) + savedBuilds.forEach { + assertNull(it[BuildTable.branch]) + assertNotNull(it[BuildTable.commitSha]) + assertNull(it[BuildTable.commitAuthor]) + assertNull(it[BuildTable.commitMessage]) + assertNull(it[BuildTable.committedAt]) + assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) + } } } @@ -119,18 +121,20 @@ class BuildsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) { ) } - val savedBuilds = BuildTable.selectAll() - .filter { it[BuildTable.groupId] == testGroup } - .filter { it[BuildTable.appId] == testApp } - .filter { it[BuildTable.buildVersion] == testBuildVersion } - assertEquals(1, savedBuilds.size) - savedBuilds.forEach { - assertEquals("main", it[BuildTable.branch]) - assertNotNull(it[BuildTable.commitSha]) - assertEquals("John Doe", it[BuildTable.commitAuthor]) - assertEquals("Initial commit", it[BuildTable.commitMessage]) - assertNotNull(it[BuildTable.committedAt]) - assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedBuilds = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testGroup } + .filter { it[BuildTable.appId] == testApp } + .filter { it[BuildTable.buildVersion] == testBuildVersion } + assertEquals(1, savedBuilds.size) + savedBuilds.forEach { + assertEquals("main", it[BuildTable.branch]) + assertNotNull(it[BuildTable.commitSha]) + assertEquals("John Doe", it[BuildTable.commitAuthor]) + assertEquals("Initial commit", it[BuildTable.commitMessage]) + assertNotNull(it[BuildTable.committedAt]) + assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) + } } } @@ -160,6 +164,22 @@ class BuildsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) { } """.trimIndent() ) + }.apply { + assertEquals(HttpStatusCode.OK, status) + } + + waitUntilInTransaction { + val savedBuilds = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testGroup } + .filter { it[BuildTable.appId] == testApp } + .filter { it[BuildTable.buildVersion] == testBuildVersion } + assertEquals(1, savedBuilds.size) + savedBuilds.forEach { + assertNotNull(it[BuildTable.branch]) + assertNotNull(it[BuildTable.commitAuthor]) + assertNotNull(it[BuildTable.commitMessage]) + assertNotNull(it[BuildTable.committedAt]) + } } app.client.put("/builds") { @@ -178,16 +198,18 @@ class BuildsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) { assertEquals(HttpStatusCode.OK, status) } - val buildsBeforeInfo = BuildTable.selectAll() - .filter { it[BuildTable.groupId] == testGroup } - .filter { it[BuildTable.appId] == testApp } - .filter { it[BuildTable.buildVersion] == testBuildVersion } - assertEquals(1, buildsBeforeInfo.size) - buildsBeforeInfo.forEach { - assertNotNull(it[BuildTable.branch]) - assertNotNull(it[BuildTable.commitAuthor]) - assertNotNull(it[BuildTable.commitMessage]) - assertNotNull(it[BuildTable.committedAt]) + waitUntilInTransaction { + val buildsBeforeInfo = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testGroup } + .filter { it[BuildTable.appId] == testApp } + .filter { it[BuildTable.buildVersion] == testBuildVersion } + assertEquals(1, buildsBeforeInfo.size) + buildsBeforeInfo.forEach { + assertNotNull(it[BuildTable.branch]) + assertNotNull(it[BuildTable.commitAuthor]) + assertNotNull(it[BuildTable.commitMessage]) + assertNotNull(it[BuildTable.committedAt]) + } } } diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/CoverageApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/CoverageApiTest.kt index ac8ffa09d..1b6cf627e 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/CoverageApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/CoverageApiTest.kt @@ -20,7 +20,6 @@ import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable import com.epam.drill.admin.test.* import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig import com.epam.drill.admin.writer.rawdata.config.rawDataServicesDIModule -import com.epam.drill.admin.writer.rawdata.table.MethodTable import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* @@ -85,17 +84,19 @@ class CoverageApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) ) } - val savedCoverageMethods = MethodCoverageTable.selectAll().asSequence() - .filter { it[MethodCoverageTable.groupId] == testGroup } - .filter { it[MethodCoverageTable.appId] == testApp } - .filter { it[MethodCoverageTable.instanceId] == testInstance } - .filter { it[MethodCoverageTable.buildId] == "$testGroup:$testApp:$testBuildVersion" } - .filter { it[MethodCoverageTable.testId] == testTestId } - .toList() - assertEquals(2, savedCoverageMethods.size) - savedCoverageMethods.forEach { - assertTrue(it[MethodCoverageTable.createdAt] >= timeBeforeTest) - assertTrue(it[MethodCoverageTable.methodId] != null) + waitUntilInTransaction { + val savedCoverageMethods = MethodCoverageTable.selectAll().asSequence() + .filter { it[MethodCoverageTable.groupId] == testGroup } + .filter { it[MethodCoverageTable.appId] == testApp } + .filter { it[MethodCoverageTable.instanceId] == testInstance } + .filter { it[MethodCoverageTable.buildId] == "$testGroup:$testApp:$testBuildVersion" } + .filter { it[MethodCoverageTable.testId] == testTestId } + .toList() + assertEquals(2, savedCoverageMethods.size) + savedCoverageMethods.forEach { + assertTrue(it[MethodCoverageTable.createdAt] >= timeBeforeTest) + assertTrue(it[MethodCoverageTable.methodId] != null) + } } } } \ No newline at end of file diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/InstancesApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/InstancesApiTest.kt index 8c2438fe9..71fcfefb1 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/InstancesApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/InstancesApiTest.kt @@ -64,7 +64,7 @@ class InstancesApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) withRollback { val testGroup = "test-group" val testApp = "test-app" - val testInstance = "test-instance" + val testInstance = "test-instance-1" val timeBeforeTest = LocalDateTime.now() val app = drillApplication(rawDataServicesDIModule) { putInstances() @@ -95,31 +95,33 @@ class InstancesApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) ) } - val savedBuilds = BuildTable.selectAll() - .filter { it[BuildTable.groupId] == testGroup } - .filter { it[BuildTable.appId] == testApp } - .filter { it[BuildTable.instanceId] == testInstance } - assertEquals(1, savedBuilds.size) - savedBuilds.forEach { - assertNotNull(it[BuildTable.commitSha]) - assertNotNull(it[BuildTable.buildVersion]) - assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) - } - val savedInstances = InstanceTable.selectAll() - .filter { it[InstanceTable.groupId] == testGroup } - .filter { it[InstanceTable.appId] == testApp } - .filter { it[InstanceTable.id].value == testInstance } - assertEquals(1, savedInstances.size) - savedInstances.forEach { - assertNotNull(it[InstanceTable.envId]) - assertTrue(it[InstanceTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedBuilds = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testGroup } + .filter { it[BuildTable.appId] == testApp } + .filter { it[BuildTable.instanceId] == testInstance } + assertEquals(1, savedBuilds.size) + savedBuilds.forEach { + assertNotNull(it[BuildTable.commitSha]) + assertNotNull(it[BuildTable.buildVersion]) + assertTrue(it[BuildTable.createdAt] >= timeBeforeTest) + } + val savedInstances = InstanceTable.selectAll() + .filter { it[InstanceTable.groupId] == testGroup } + .filter { it[InstanceTable.appId] == testApp } + .filter { it[InstanceTable.id].value == testInstance } + assertEquals(1, savedInstances.size) + savedInstances.forEach { + assertNotNull(it[InstanceTable.envId]) + assertTrue(it[InstanceTable.createdAt] >= timeBeforeTest) + } } } @Test fun `given existing build, put instances service should refer to existing build, save new instance in database and return OK`() = withRollback { - val testInstance = "test-instance" + val testInstance = "test-instance-2" val timeBeforeTest = LocalDateTime.now() val app = drillApplication(rawDataServicesDIModule) { putInstances() @@ -149,19 +151,21 @@ class InstancesApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) ) } - val savedInstances = InstanceTable.selectAll() - .filter { it[InstanceTable.groupId] == testExistingGroup } - .filter { it[InstanceTable.appId] == testExistingApp } - .filter { it[InstanceTable.id].value == testInstance } - assertEquals(1, savedInstances.size) - savedInstances.forEach { - assertNotNull(it[InstanceTable.envId]) - assertTrue(it[InstanceTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedInstances = InstanceTable.selectAll() + .filter { it[InstanceTable.groupId] == testExistingGroup } + .filter { it[InstanceTable.appId] == testExistingApp } + .filter { it[InstanceTable.id].value == testInstance } + assertEquals(1, savedInstances.size) + savedInstances.forEach { + assertNotNull(it[InstanceTable.envId]) + assertTrue(it[InstanceTable.createdAt] >= timeBeforeTest) + } + val savedBuilds = BuildTable.selectAll() + .filter { it[BuildTable.groupId] == testExistingGroup } + .filter { it[BuildTable.appId] == testExistingApp } + .filter { it[BuildTable.buildVersion] == testExistingBuildVersion } + assertEquals(1, savedBuilds.size) } - val savedBuilds = BuildTable.selectAll() - .filter { it[BuildTable.groupId] == testExistingGroup } - .filter { it[BuildTable.appId] == testExistingApp } - .filter { it[BuildTable.buildVersion] == testExistingBuildVersion } - assertEquals(1, savedBuilds.size) } } \ No newline at end of file diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/MethodsApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/MethodsApiTest.kt index 891b6662a..923a941c1 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/MethodsApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/MethodsApiTest.kt @@ -89,13 +89,15 @@ class MethodsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }) { ) } - val savedMethods = MethodTable.selectAll() - .filter { it[MethodTable.groupId] == testGroup } - .filter { it[MethodTable.appId] == testApp } - .filter { it[MethodTable.classname] == testClassname } - assertEquals(2, savedMethods.size) - savedMethods.forEach { - assertTrue(it[MethodTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedMethods = MethodTable.selectAll() + .filter { it[MethodTable.groupId] == testGroup } + .filter { it[MethodTable.appId] == testApp } + .filter { it[MethodTable.classname] == testClassname } + assertEquals(2, savedMethods.size) + savedMethods.forEach { + assertTrue(it[MethodTable.createdAt] >= timeBeforeTest) + } } } } \ No newline at end of file diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/SettingsRoutesTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/SettingsRoutesTest.kt index 83949cef3..4aa5cf612 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/SettingsRoutesTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/SettingsRoutesTest.kt @@ -66,7 +66,7 @@ class SettingsRoutesTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) }.apply { assertEquals(HttpStatusCode.OK, status) } - transaction { + waitUntilInTransaction { val savedSettings = GroupSettingsTable.selectAll().where { GroupSettingsTable.id eq testGroup }.single() assertEquals(30, savedSettings[GroupSettingsTable.retentionPeriodDays]) assertEquals(10, savedSettings[GroupSettingsTable.metricsPeriodDays]) @@ -112,7 +112,7 @@ class SettingsRoutesTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) client.delete("group-settings/$testGroup").apply { assertEquals(HttpStatusCode.OK, status) } - transaction { + waitUntilInTransaction { assertTrue(GroupSettingsTable.selectAll().where { GroupSettingsTable.id eq testGroup }.empty()) } } diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt index 909dcd752..b1e6a2bcc 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt @@ -18,6 +18,7 @@ package com.epam.drill.admin.writer.rawdata import com.epam.drill.admin.test.DatabaseTests import com.epam.drill.admin.test.assertJsonEquals import com.epam.drill.admin.test.drillApplication +import com.epam.drill.admin.test.waitUntilInTransaction import com.epam.drill.admin.test.withRollback import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig import com.epam.drill.admin.writer.rawdata.config.rawDataServicesDIModule @@ -109,28 +110,30 @@ class TestMetadataApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) ) } - val savedTestLaunches = TestLaunchTable.selectAll() - .filter { it[TestLaunchTable.groupId] == testGroup } - .filter { it[TestLaunchTable.testSessionId] == testSession } - .filter { it[TestLaunchTable.testDefinitionId] == testDefinition } - assertEquals(2, savedTestLaunches.size) - savedTestLaunches.forEach { - assertNotNull(it[TestLaunchTable.duration]) - assertNotNull(it[TestLaunchTable.result]) - assertTrue(it[TestLaunchTable.createdAt] >= timeBeforeTest) - } + waitUntilInTransaction { + val savedTestLaunches = TestLaunchTable.selectAll() + .filter { it[TestLaunchTable.groupId] == testGroup } + .filter { it[TestLaunchTable.testSessionId] == testSession } + .filter { it[TestLaunchTable.testDefinitionId] == testDefinition } + assertEquals(2, savedTestLaunches.size) + savedTestLaunches.forEach { + assertNotNull(it[TestLaunchTable.duration]) + assertNotNull(it[TestLaunchTable.result]) + assertTrue(it[TestLaunchTable.createdAt] >= timeBeforeTest) + } - val savedTestDefinitions = TestDefinitionTable.selectAll() - .filter { it[TestDefinitionTable.groupId] == testGroup } - .filter { it[TestDefinitionTable.id].value == testDefinition } - assertEquals(1, savedTestDefinitions.size) - savedTestDefinitions.forEach { - assertNotNull(it[TestDefinitionTable.runner]) - assertNotNull(it[TestDefinitionTable.name]) - assertNotNull(it[TestDefinitionTable.path]) - assertEquals(2, it[TestDefinitionTable.tags]?.size) - assertEquals(2, it[TestDefinitionTable.metadata]?.jsonObject?.size) - assertTrue(it[TestDefinitionTable.createdAt] >= timeBeforeTest) + val savedTestDefinitions = TestDefinitionTable.selectAll() + .filter { it[TestDefinitionTable.groupId] == testGroup } + .filter { it[TestDefinitionTable.id].value == testDefinition } + assertEquals(1, savedTestDefinitions.size) + savedTestDefinitions.forEach { + assertNotNull(it[TestDefinitionTable.runner]) + assertNotNull(it[TestDefinitionTable.name]) + assertNotNull(it[TestDefinitionTable.path]) + assertEquals(2, it[TestDefinitionTable.tags]?.size) + assertEquals(2, it[TestDefinitionTable.metadata]?.jsonObject?.size) + assertTrue(it[TestDefinitionTable.createdAt] >= timeBeforeTest) + } } } @@ -186,14 +189,16 @@ class TestMetadataApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) ) } - val saved = TestDefinitionTable.selectAll() - .filter { it[TestDefinitionTable.groupId] == testGroup } + waitUntilInTransaction { + val saved = TestDefinitionTable.selectAll() + .filter { it[TestDefinitionTable.groupId] == testGroup } - assertEquals(2, saved.size) - saved.forEach { - assertNotNull(it[TestDefinitionTable.runner]) - assertNotNull(it[TestDefinitionTable.name]) - assertTrue(it[TestDefinitionTable.createdAt] >= timeBeforeTest) + assertEquals(2, saved.size) + saved.forEach { + assertNotNull(it[TestDefinitionTable.runner]) + assertNotNull(it[TestDefinitionTable.name]) + assertTrue(it[TestDefinitionTable.createdAt] >= timeBeforeTest) + } } } @@ -246,16 +251,18 @@ class TestMetadataApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) ) } - val saved = TestLaunchTable.selectAll() - .filter { it[TestLaunchTable.groupId] == testGroup } - .filter { it[TestLaunchTable.testSessionId] == testSession } - .filter { it[TestLaunchTable.testDefinitionId] == testDefinition } + waitUntilInTransaction { + val saved = TestLaunchTable.selectAll() + .filter { it[TestLaunchTable.groupId] == testGroup } + .filter { it[TestLaunchTable.testSessionId] == testSession } + .filter { it[TestLaunchTable.testDefinitionId] == testDefinition } - assertEquals(2, saved.size) - saved.forEach { - assertNotNull(it[TestLaunchTable.result]) - assertNotNull(it[TestLaunchTable.duration]) - assertTrue(it[TestLaunchTable.createdAt] >= timeBeforeTest) + assertEquals(2, saved.size) + saved.forEach { + assertNotNull(it[TestLaunchTable.result]) + assertNotNull(it[TestLaunchTable.duration]) + assertTrue(it[TestLaunchTable.createdAt] >= timeBeforeTest) + } } } diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestSessionsApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestSessionsApiTest.kt index 2dabb66f0..8ae0e8dcf 100644 --- a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestSessionsApiTest.kt +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestSessionsApiTest.kt @@ -66,14 +66,16 @@ class TestSessionsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) ) } - val savedTestSessions = TestSessionTable.selectAll() - .filter { it[TestSessionTable.groupId] == testGroup } - .filter { it[TestSessionTable.id].value == testSession } - assertEquals(1, savedTestSessions.size) - savedTestSessions.forEach { - assertNotNull(it[TestSessionTable.testTaskId]) - assertNotNull(it[TestSessionTable.startedAt]) - assertTrue(it[TestSessionTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedTestSessions = TestSessionTable.selectAll() + .filter { it[TestSessionTable.groupId] == testGroup } + .filter { it[TestSessionTable.id].value == testSession } + assertEquals(1, savedTestSessions.size) + savedTestSessions.forEach { + assertNotNull(it[TestSessionTable.testTaskId]) + assertNotNull(it[TestSessionTable.startedAt]) + assertTrue(it[TestSessionTable.createdAt] >= timeBeforeTest) + } } } @@ -119,13 +121,15 @@ class TestSessionsApiTest : DatabaseTests({ RawDataWriterDatabaseConfig.init(it) ) } - val savedSessionBuilds = TestSessionBuildTable.selectAll() - .filter { it[TestSessionBuildTable.testSessionId] == testSession } - assertEquals(2, savedSessionBuilds.size) - savedSessionBuilds.forEach { - assertNotNull(it[TestSessionBuildTable.buildId]) - assertNotNull(it[TestSessionBuildTable.groupId]) - assertTrue(it[TestSessionBuildTable.createdAt] >= timeBeforeTest) + waitUntilInTransaction { + val savedSessionBuilds = TestSessionBuildTable.selectAll() + .filter { it[TestSessionBuildTable.testSessionId] == testSession } + assertEquals(2, savedSessionBuilds.size) + savedSessionBuilds.forEach { + assertNotNull(it[TestSessionBuildTable.buildId]) + assertNotNull(it[TestSessionBuildTable.groupId]) + assertTrue(it[TestSessionBuildTable.createdAt] >= timeBeforeTest) + } } } } \ No newline at end of file diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueueTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueueTest.kt new file mode 100644 index 000000000..384e0b8fa --- /dev/null +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/ChannelDataQueueTest.kt @@ -0,0 +1,81 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.impl + +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import com.epam.drill.admin.writer.rawdata.queue.QueueInput +import com.epam.drill.admin.writer.rawdata.route.BuildsRoute +import com.epam.drill.admin.writer.rawdata.route.jsonConfig +import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ClosedSendChannelException +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.mockito.kotlin.mock +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.time.Duration.Companion.seconds + +class ChannelDataQueueTest { + + @Test + fun `enqueue should deserialize payload and make it available via dequeue`() { + val queue = ChannelDataQueue( + deserializer = ::json, + routeToPayloadType = { BuildPayload::class }, + capacity = Channel.UNLIMITED, + shutdownTimeout = 1.seconds, + metrics = RawDataMeter(SimpleMeterRegistry()), + ) + val testBytes = BuildPayload(groupId = "my-group", appId = "my-app", buildVersion = "1.0.0").toBytes() + val testMetadata = mapOf("key-1" to "value-1") + + runBlocking { + queue.enqueue(QueueInput(BuildsRoute(), testBytes, testMetadata)) + val output = withTimeout(2_000) { queue.dequeue() } + + assertEquals("my-group", output.payload.groupId) + assertEquals("my-app", output.payload.appId) + assertEquals(testMetadata, output.metadata) + } + + queue.close() + } + + @Test + fun `close should close the output channel and prevent further enqueueing`() { + val queue = ChannelDataQueue( + deserializer = ::json, + routeToPayloadType = { BuildPayload::class }, + capacity = Channel.UNLIMITED, + shutdownTimeout = 1.seconds, + metrics = RawDataMeter(SimpleMeterRegistry()), + ) + + queue.close() + + runBlocking { + assertFailsWith { + queue.enqueue(QueueInput(BuildsRoute(), ByteArray(0), emptyMap())) + } + } + } + + private fun BuildPayload.toBytes() = + jsonConfig.encodeToString(BuildPayload.serializer(), this).toByteArray(Charsets.UTF_8) +} \ No newline at end of file diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueueTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueueTest.kt new file mode 100644 index 000000000..e48e05f52 --- /dev/null +++ b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/queue/impl/KafkaDataQueueTest.kt @@ -0,0 +1,111 @@ +/** + * Copyright 2020 - 2022 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.epam.drill.admin.writer.rawdata.queue.impl + +import com.epam.drill.admin.writer.rawdata.config.RawDataMeter +import com.epam.drill.admin.writer.rawdata.queue.QueueInput +import com.epam.drill.admin.writer.rawdata.route.BuildsRoute +import com.epam.drill.admin.writer.rawdata.route.jsonConfig +import com.epam.drill.admin.writer.rawdata.route.payload.BuildPayload +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.testcontainers.kafka.ConfluentKafkaContainer +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers +import org.testcontainers.utility.DockerImageName +import java.util.UUID +import kotlin.reflect.KClass +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +@Testcontainers +class KafkaDataQueueTest { + + companion object { + @Container + @JvmField + val kafka: ConfluentKafkaContainer = + ConfluentKafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.4.0")) + } + + private fun uniqueTopic() = "test-topic-${UUID.randomUUID()}" + private fun uniqueGroupId() = "test-group-${UUID.randomUUID()}" + + private fun createQueue( + topic: String = uniqueTopic(), + recordKeyToPayloadType: (String) -> KClass = { + when (it) { + "builds" -> BuildPayload::class + else -> throw IllegalArgumentException("Unknown record key: $it") + } + }, + routeToRecordKey: (BuildsRoute) -> String = { "builds" }, + ) = KafkaDataQueue.create( + bootstrapServers = kafka.bootstrapServers, + topic = topic, + consumerGroupId = uniqueGroupId(), + deserializer = ::json, + recordKeyToPayloadType = recordKeyToPayloadType, + routeToRecordKey = routeToRecordKey, + capacity = Channel.UNLIMITED, + pollTimeout = 500.milliseconds, + shutdownTimeout = 5.seconds, + metrics = RawDataMeter(SimpleMeterRegistry()), + ) + + + @Test + fun `enqueue and dequeue should deliver payload`() { + val queue = createQueue() + val testBytes = BuildPayload(groupId = "my-group", appId = "my-app", buildVersion = "1.0.0").toBytes() + val testMetadata = mapOf("key-1" to "value-1") + + runBlocking { + queue.enqueue(QueueInput(BuildsRoute(), testBytes, testMetadata)) + val output = withTimeout(10_000) { queue.dequeue() } + + assertEquals("my-group", output.payload.groupId) + assertEquals("my-app", output.payload.appId) + assertEquals("value-1", output.metadata["key-1"]) + } + + queue.close() + } + + @Test + fun `close should stop consuming and mark channel as closed`() { + val queue = createQueue() + + queue.close() + + runBlocking { + assertFailsWith("Cannot perform operation after producer has been closed") { + queue.enqueue(QueueInput(BuildsRoute(), ByteArray(0), emptyMap())) + } + } + } + + private fun BuildPayload.toBytes() = + jsonConfig.encodeToString(BuildPayload.serializer(), this).toByteArray(Charsets.UTF_8) +} + + + diff --git a/build.gradle.kts b/build.gradle.kts index 5c411930a..6b5026764 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -49,6 +49,11 @@ if(version == Project.DEFAULT_VERSION) { } subprojects { + plugins.withId("org.jetbrains.kotlin.jvm") { + configure { + jvmToolchain(17) + } + } val constraints = setOf( dependencies.constraints.create("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"), dependencies.constraints.create("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"), @@ -66,6 +71,8 @@ subprojects { dependencies.constraints.create("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:$kotlinxSerializationVersion"), ) configurations.all { - dependencyConstraints += constraints + if (isCanBeDeclared) { + dependencyConstraints += constraints + } } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 4134aff76..5de40ac4d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,26 +1,29 @@ -kotlinVersion = 2.0.0 +kotlinVersion = 2.1.21 licenseVersion = 0.16.1 -kotlinxCoroutinesVersion = 1.7.3 -kotlinxSerializationVersion = 1.6.2 -kotlinxDatetimeVersion = 0.4.1 -exposedVersion = 0.51.1 +kotlinxCoroutinesVersion = 1.10.1 +kotlinxSerializationVersion = 1.8.0 +kotlinxDatetimeVersion = 0.6.1 +exposedVersion = 0.61.0 grgitVersion = 4.1.1 -ktorVersion = 2.3.11 -kodeinVersion = 7.21.1 +ktorVersion = 3.1.3 +kodeinVersion = 7.26.1 microutilsLoggingVersion = 3.0.5 postgresSqlVersion = 42.3.8 testContainersVersion = 1.21.4 junitJupiterVersion = 5.8.1 zaxxerHikaricpVersion = 4.0.3 -shadowPluginVersion = 7.1.0 +shadowPluginVersion = 8.1.1 flywaydbVersion = 8.4.1 jibVersion = 3.1.4 openApiGeneratorVersion = 6.6.0 mockitoKotlinVersion = 4.1.0 +awaitilityVersion = 4.2.2 jbcryptVersion = 0.4 caffeineVersion = 2.9.3 quartzVersion = 2.5.0 logbackVersion = 1.3.14 +kafkaClientsVersion = 3.7.0 +micrometerVersion = 1.12.13 loggerSkipJvmTests = false testsSkipIntegrationTests = false diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists