diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 4b8854887c1966..ff716e06080d66 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -54,7 +54,7 @@ org.eclipse.jetty jetty-server - test + compile org.eclipse.jetty diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 7cc70c493c0f66..1b4bba7c1dd41d 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -649,6 +649,7 @@ && getMaxInactiveInterval() > 0) { */ protected void doFilter(FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + JettyAuthenticationHelper.publishRemoteUser(request); filterChain.doFilter(request, response); } diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JettyAuthenticationHelper.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JettyAuthenticationHelper.java new file mode 100644 index 00000000000000..fb534db99292ad --- /dev/null +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JettyAuthenticationHelper.java @@ -0,0 +1,169 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.security.authentication.server; + +import java.security.Principal; +import java.util.Collections; +import javax.security.auth.Subject; +import javax.servlet.ServletRequest; +import javax.servlet.http.HttpServletRequest; +import org.eclipse.jetty.server.Authentication; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.UserIdentity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Publishes the authenticated user on the underlying Jetty {@link Request} so + * the access log %u can resolve it. Hadoop's auth filters expose the user via + * an {@code HttpServletRequestWrapper}, which Jetty's request log handler does + * not see; pushing the authentication onto the base request makes the user + * visible after the filter chain returns. + */ +public final class JettyAuthenticationHelper { + private static final Logger LOG = LoggerFactory.getLogger(JettyAuthenticationHelper.class); + + private JettyAuthenticationHelper() { + } + + /** + * Publishes {@code request.getRemoteUser()} as the authenticated user on + * the underlying Jetty request. First writer wins so that callers that + * resolve the effective user earliest (e.g. delegation-token handler with + * the doAs user) are not overwritten by later filter-chain hooks. No-op + * when there is no remote user, when the request is not running on Jetty, + * or when the base request already has an {@link Authentication.User}. + * + * @param request the wrapped HTTP request, after the auth filter has set + * the remote user + */ + public static void publishRemoteUser(HttpServletRequest request) { + if (request == null) { + return; + } + String user = request.getRemoteUser(); + publishRemoteUser(request, user); + } + + /** + * Same as {@link #publishRemoteUser(HttpServletRequest)} but uses the + * provided user name instead of {@code request.getRemoteUser()}. Use this + * when the effective user (e.g. doAs) is not yet reflected on the request. + * + * @param request the HTTP request used to find the underlying Jetty request + * @param user the user name to publish + */ + public static void publishRemoteUser(HttpServletRequest request, String user) { + if (user == null || user.isEmpty()) { + return; + } + Request base = Request.getBaseRequest(request); + if (base == null) { + return; + } + + Authentication existing = base.getAuthentication(); + if (existing instanceof Authentication.User) { + if (LOG.isDebugEnabled()) { + LOG.debug("publishRemoteUser skipped: already published existing='{}', incoming='{}'", + ((Authentication.User) existing).getUserIdentity() + .getUserPrincipal().getName(), user); + } + return; + } + LOG.debug("publishRemoteUser published user='{}'", user); + base.setAuthentication(new RemoteUserAuthentication(user)); + } + + private static final class RemoteUserAuthentication + implements Authentication.User { + private final UserIdentity identity; + + RemoteUserAuthentication(String name) { + Principal principal = new RemoteUserPrincipal(name); + Subject subject = new Subject(true, + Collections.singleton(principal), + Collections.emptySet(), + Collections.emptySet()); + this.identity = new RemoteUserIdentity(subject, principal); + } + + @Override + public String getAuthMethod() { + return "HADOOP"; + } + + @Override + public UserIdentity getUserIdentity() { + return identity; + } + + @Override + public boolean isUserInRole(UserIdentity.Scope scope, String role) { + return false; + } + + @Override + public void logout() { + } + + @Override + public Authentication logout(ServletRequest request) { + return Authentication.UNAUTHENTICATED; + } + } + + private static final class RemoteUserIdentity implements UserIdentity { + private final Subject subject; + private final Principal principal; + + RemoteUserIdentity(Subject subject, Principal principal) { + this.subject = subject; + this.principal = principal; + } + + @Override + public Subject getSubject() { + return subject; + } + + @Override + public Principal getUserPrincipal() { + return principal; + } + + @Override + public boolean isUserInRole(String role, Scope scope) { + return false; + } + } + + private static final class RemoteUserPrincipal implements Principal { + private final String name; + + RemoteUserPrincipal(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJettyAuthenticationHelper.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJettyAuthenticationHelper.java new file mode 100644 index 00000000000000..586f4c90072a4a --- /dev/null +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJettyAuthenticationHelper.java @@ -0,0 +1,95 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ +package org.apache.hadoop.security.authentication.server; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import org.eclipse.jetty.server.Authentication; +import org.eclipse.jetty.server.Request; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestJettyAuthenticationHelper { + + private static HttpServletRequest wrap(Request base, String remoteUser) { + return new HttpServletRequestWrapper(base) { + @Override + public String getRemoteUser() { + return remoteUser; + } + }; + } + + @Test + public void testSetsAuthenticationFromRemoteUser() { + Request base = new Request(null, null); + JettyAuthenticationHelper.publishRemoteUser(wrap(base, "alice")); + + Authentication auth = base.getAuthentication(); + assertInstanceOf(Authentication.User.class, auth); + Authentication.User user = (Authentication.User) auth; + assertEquals("alice", user.getUserIdentity().getUserPrincipal().getName()); + } + + @Test + public void testNullRemoteUserIsNoOp() { + Request base = new Request(null, null); + JettyAuthenticationHelper.publishRemoteUser(wrap(base, null)); + assertNull(base.getAuthentication()); + } + + @Test + public void testEmptyRemoteUserIsNoOp() { + Request base = new Request(null, null); + JettyAuthenticationHelper.publishRemoteUser(wrap(base, "")); + assertNull(base.getAuthentication()); + } + + @Test + public void testPreservesExistingAuthentication() { + Request base = new Request(null, null); + JettyAuthenticationHelper.publishRemoteUser(wrap(base, "eve")); + Authentication existing = base.getAuthentication(); + + JettyAuthenticationHelper.publishRemoteUser(wrap(base, "bob")); + + assertSame(existing, base.getAuthentication()); + } + + @Test + public void testExplicitUserOverloadSetsAuthentication() { + Request base = new Request(null, null); + HttpServletRequest wrapped = wrap(base, null); + + JettyAuthenticationHelper.publishRemoteUser(wrapped, "dave"); + + Authentication auth = base.getAuthentication(); + assertInstanceOf(Authentication.User.class, auth); + assertEquals("dave", ((Authentication.User) auth) + .getUserIdentity().getUserPrincipal().getName()); + } + + @Test + public void testNonJettyRequestIsNoOp() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn("carol"); + JettyAuthenticationHelper.publishRemoteUser(request); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java index f4ede6f35edb0c..54671fd8ddb603 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java @@ -39,6 +39,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.server.AuthenticationHandler; import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.JettyAuthenticationHelper; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.authorize.ProxyUsers; @@ -263,6 +264,9 @@ public boolean managementOperation(AuthenticationToken token, return false; } } + if (requestUgi != null) { + JettyAuthenticationHelper.publishRemoteUser(request, requestUgi.getShortUserName()); + } Map map = null; switch (dtOp) { case GETDELEGATIONTOKEN: diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer2AccessLogUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer2AccessLogUser.java new file mode 100644 index 00000000000000..6df48ab5f7d2cb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer2AccessLogUser.java @@ -0,0 +1,242 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.http; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.Principal; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.authentication.server.JettyAuthenticationHelper; +import org.apache.hadoop.security.authentication.server.ProxyUserAuthenticationFilterInitializer; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.GenericTestUtils.LogCapturer; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies that the HttpServer2 access log records the authenticated user. + * A test filter wraps the request with a fake remote user and publishes it + * via {@link JettyAuthenticationHelper} so the access log %u picks it up. + */ +public class TestHttpServer2AccessLogUser extends HttpServerFunctionalTest { + + private static final String EXPECTED_USER = "alice"; + + private HttpServer2 server; + private LogCapturer accessLogCapturer; + + @AfterEach + public void tearDown() throws Exception { + if (accessLogCapturer != null) { + accessLogCapturer.stopCapturing(); + } + if (server != null && server.isAlive()) { + server.stop(); + } + } + + @Test + public void testAccessLogIncludesAuthenticatedUser() throws Exception { + org.apache.log4j.Logger accessLogger = + LogManager.getLogger("http.requests.test"); + accessLogger.setLevel(Level.ALL); + accessLogCapturer = LogCapturer.captureLogs(accessLogger); + + Configuration conf = new Configuration(); + server = createTestServer(conf); + server.addGlobalFilter("testAuth", FakeAuthFilter.class.getName(), null); + server.start(); + baseUrl = getServerURL(server); + + HttpURLConnection conn = + (HttpURLConnection) new URL(baseUrl, "/jmx").openConnection(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + conn.disconnect(); + + GenericTestUtils.waitFor(() -> { + String out = accessLogCapturer.getOutput(); + return out != null && !out.isEmpty(); + }, 100, 5000); + + String captured = accessLogCapturer.getOutput(); + assertTrue(captured.contains(" " + EXPECTED_USER + " "), + "Access log should contain user '" + EXPECTED_USER + + "', but was: " + captured); + } + + @Test + public void testAccessLogIncludesUserFromAuthenticationFilter() throws Exception { + org.apache.log4j.Logger accessLogger = + LogManager.getLogger("http.requests.test"); + accessLogger.setLevel(Level.ALL); + accessLogCapturer = LogCapturer.captureLogs(accessLogger); + + String authPrefix = "hadoop.http.authentication."; + Configuration conf = new Configuration(); + conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, + AuthenticationFilterInitializer.class.getName()); + conf.set(authPrefix + "type", "simple"); + conf.set(authPrefix + PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, + "false"); + + server = createTestServer(conf); + server.start(); + baseUrl = getServerURL(server); + + HttpURLConnection conn = (HttpURLConnection) new URL( + baseUrl, "/jmx?user.name=" + EXPECTED_USER).openConnection(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + conn.disconnect(); + + GenericTestUtils.waitFor(() -> { + String out = accessLogCapturer.getOutput(); + return out != null && !out.isEmpty(); + }, 100, 5000); + + String captured = accessLogCapturer.getOutput(); + assertTrue(captured.contains(" " + EXPECTED_USER + " "), + "Access log should contain user '" + EXPECTED_USER + + "' from AuthenticationFilter, but was: " + captured); + } + + @Test + public void testAccessLogIncludesDoAsUserViaProxyUserFilter() throws Exception { + org.apache.log4j.Logger accessLogger = + LogManager.getLogger("http.requests.test"); + accessLogger.setLevel(Level.ALL); + accessLogCapturer = LogCapturer.captureLogs(accessLogger); + + String realUser = "alice"; + String doAsUser = "bob"; + + String authPrefix = "hadoop.http.authentication."; + Configuration conf = new Configuration(); + conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, + ProxyUserAuthenticationFilterInitializer.class.getName()); + conf.set(authPrefix + "type", "simple"); + conf.set(authPrefix + PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, + "false"); + // Allow alice to impersonate any user from any host. + conf.set("hadoop.proxyuser." + realUser + ".groups", "*"); + conf.set("hadoop.proxyuser." + realUser + ".hosts", "*"); + + server = createTestServer(conf); + server.start(); + baseUrl = getServerURL(server); + + URL url = new URL(baseUrl, + "/jmx?user.name=" + realUser + "&doas=" + doAsUser); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + conn.disconnect(); + + GenericTestUtils.waitFor(() -> { + String out = accessLogCapturer.getOutput(); + return out != null && !out.isEmpty(); + }, 100, 5000); + + String captured = accessLogCapturer.getOutput(); + assertTrue(captured.contains(" " + doAsUser + " "), + "Access log should contain doAs user '" + doAsUser + + "', but was: " + captured); + assertFalse(captured.contains(" " + realUser + " "), + "Access log should NOT contain real user '" + realUser + + "' when doAs is applied, but was: " + captured); + } + + @Test + public void testAccessLogShowsDashWhenNoUser() throws Exception { + org.apache.log4j.Logger accessLogger = + LogManager.getLogger("http.requests.test"); + accessLogger.setLevel(Level.ALL); + accessLogCapturer = LogCapturer.captureLogs(accessLogger); + + Configuration conf = new Configuration(); + server = createTestServer(conf); + server.start(); + baseUrl = getServerURL(server); + + HttpURLConnection conn = + (HttpURLConnection) new URL(baseUrl, "/jmx").openConnection(); + assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); + conn.disconnect(); + + GenericTestUtils.waitFor(() -> { + String out = accessLogCapturer.getOutput(); + return out != null && !out.isEmpty(); + }, 100, 5000); + + String captured = accessLogCapturer.getOutput(); + assertTrue(captured.contains(" - - "), + "Access log should show '-' for unauthenticated user, but was: " + + captured); + } + + /** + * Test-only filter that wraps the request with a fixed remote user and + * pushes it onto the underlying Jetty request so the access log %u + * resolves to the user. + */ + public static class FakeAuthFilter implements Filter { + @Override + public void init(FilterConfig filterConfig) { + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) + throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletRequest wrapped = new HttpServletRequestWrapper(httpRequest) { + @Override + public String getRemoteUser() { + return EXPECTED_USER; + } + + @Override + public Principal getUserPrincipal() { + return () -> EXPECTED_USER; + } + }; + JettyAuthenticationHelper.publishRemoteUser(wrapped); + chain.doFilter(wrapped, response); + } + + @Override + public void destroy() { + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index 4265c288e88d98..96811fd0b208c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.common; +import org.apache.hadoop.security.authentication.server.JettyAuthenticationHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -145,7 +146,9 @@ public static UserGroupInformation getUGI(ServletContext context, ProxyUsers.authorize(ugi, getRemoteAddr(request)); } } - + if (ugi != null) { + JettyAuthenticationHelper.publishRemoteUser(request, ugi.getShortUserName()); + } if(LOG.isDebugEnabled()) LOG.debug("getUGI is returning: " + ugi.getShortUserName()); return ugi;