From bd1cd4a3b5beb62ee31f26c79c215a358a48e2f9 Mon Sep 17 00:00:00 2001 From: "Jeffrey(Xilang) Yan" <7855100+yantzu@users.noreply.github.com> Date: Mon, 26 Aug 2019 15:33:09 +0800 Subject: [PATCH 1/4] add spark.livy.owner config to spark context --- .../livy/server/batch/BatchSession.scala | 4 +-- .../interactive/InteractiveSession.scala | 4 +-- .../org/apache/livy/sessions/Session.scala | 5 +++- .../apache/livy/sessions/SessionSpec.scala | 26 +++++++++++++------ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala b/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala index 2a55c0493..63f0f4fa9 100644 --- a/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala +++ b/server/src/main/scala/org/apache/livy/server/batch/BatchSession.scala @@ -69,8 +69,8 @@ object BatchSession extends Logging { val conf = SparkApp.prepareSparkConf( appTag, livyConf, - prepareConf( - request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf)) + prepareConf(owner, request.conf, + request.jars, request.files, request.archives, request.pyFiles, livyConf)) require(request.file != null, "File is required.") val builder = new SparkProcessBuilder(livyConf) diff --git a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala index bccdb4d92..204f8d403 100644 --- a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala +++ b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala @@ -79,8 +79,8 @@ object InteractiveSession extends Logging { val impersonatedUser = accessManager.checkImpersonation(proxyUser, owner) val client = mockClient.orElse { - val conf = SparkApp.prepareSparkConf(appTag, livyConf, prepareConf( - request.conf, request.jars, request.files, request.archives, request.pyFiles, livyConf)) + val conf = SparkApp.prepareSparkConf(appTag, livyConf, prepareConf(owner, request.conf, + request.jars, request.files, request.archives, request.pyFiles, livyConf)) val builderProperties = prepareBuilderProp(conf, request.kind, livyConf) diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala b/server/src/main/scala/org/apache/livy/sessions/Session.scala index 67f78d4d2..4a8c7696b 100644 --- a/server/src/main/scala/org/apache/livy/sessions/Session.scala +++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala @@ -48,6 +48,7 @@ object Session { * - Verify that file URIs don't reference non-whitelisted local resources */ def prepareConf( + owner: String, conf: Map[String, String], jars: Seq[String], files: Seq[String], @@ -90,7 +91,9 @@ object Session { val masterConfList = Map(LivyConf.SPARK_MASTER -> livyConf.sparkMaster()) ++ livyConf.sparkDeployMode().map(LivyConf.SPARK_DEPLOY_MODE -> _).toMap - conf ++ masterConfList ++ merged + val ownerConf = Map("spark.livy.owner" -> (if (owner == null) "" else owner)) + + conf ++ masterConfList ++ merged ++ ownerConf } /** diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala index 05b41bb05..cfafcd200 100644 --- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala +++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala @@ -59,30 +59,40 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { conf.set(LivyConf.LOCAL_FS_WHITELIST, "/allowed") // Test baseline. - assert(Session.prepareConf(Map(), Nil, Nil, Nil, Nil, conf) === Map("spark.master" -> "local")) + assert(Session.prepareConf("", Map(), Nil, Nil, Nil, Nil, conf) + === Map("spark.master" -> "local")) // Test validations. intercept[IllegalArgumentException] { - Session.prepareConf(Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil, conf) + Session.prepareConf("", Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil, conf) } conf.sparkFileLists.foreach { key => intercept[IllegalArgumentException] { - Session.prepareConf(Map(key -> "file:/not_allowed"), Nil, Nil, Nil, Nil, conf) + Session.prepareConf("", Map(key -> "file:/not_allowed"), Nil, Nil, Nil, Nil, conf) } } intercept[IllegalArgumentException] { - Session.prepareConf(Map(), Seq("file:/not_allowed"), Nil, Nil, Nil, conf) + Session.prepareConf("", Map(), Seq("file:/not_allowed"), Nil, Nil, Nil, conf) } intercept[IllegalArgumentException] { - Session.prepareConf(Map(), Nil, Seq("file:/not_allowed"), Nil, Nil, conf) + Session.prepareConf("", Map(), Nil, Seq("file:/not_allowed"), Nil, Nil, conf) } intercept[IllegalArgumentException] { - Session.prepareConf(Map(), Nil, Nil, Seq("file:/not_allowed"), Nil, conf) + Session.prepareConf("", Map(), Nil, Nil, Seq("file:/not_allowed"), Nil, conf) } intercept[IllegalArgumentException] { - Session.prepareConf(Map(), Nil, Nil, Nil, Seq("file:/not_allowed"), conf) + Session.prepareConf("", Map(), Nil, Nil, Nil, Seq("file:/not_allowed"), conf) } + //Test owner + assert(Session.prepareConf("MyOwner", Map(), Nil, Nil, Nil, Nil, conf) + .get("spark.livy.owner") === "MyOwner") + assert(Session.prepareConf("", Map(), Nil, Nil, Nil, Nil, conf) + .get("spark.livy.owner") === "") + assert(Session.prepareConf(null, Map(), Nil, Nil, Nil, Nil, conf) + .get("spark.livy.owner") === "") + + // Test that file lists are merged and resolved. val base = "/file1.txt" val other = Seq("/file2.txt") @@ -91,7 +101,7 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { val userLists = Seq(LivyConf.SPARK_JARS, LivyConf.SPARK_FILES, LivyConf.SPARK_ARCHIVES, LivyConf.SPARK_PY_FILES) val baseConf = userLists.map { key => (key -> base) }.toMap - val result = Session.prepareConf(baseConf, other, other, other, other, conf) + val result = Session.prepareConf("", baseConf, other, other, other, other, conf) userLists.foreach { key => assert(result.get(key) === expected) } } From d332de0d39c6294fffaf051e031c0004d4ab76c0 Mon Sep 17 00:00:00 2001 From: "Jeffrey(Xilang) Yan" <7855100+yantzu@users.noreply.github.com> Date: Tue, 27 Aug 2019 08:35:40 +0800 Subject: [PATCH 2/4] fix ut --- .../src/test/scala/org/apache/livy/sessions/SessionSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala index cfafcd200..bd97bc3a4 100644 --- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala +++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala @@ -60,7 +60,7 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { // Test baseline. assert(Session.prepareConf("", Map(), Nil, Nil, Nil, Nil, conf) - === Map("spark.master" -> "local")) + === Map("spark.master" -> "local", "spark.livy.owner" -> "")) // Test validations. intercept[IllegalArgumentException] { From c0e52e9935a3401beb834f9ef55047c7a0532cc8 Mon Sep 17 00:00:00 2001 From: "Jeffrey(Xilang) Yan" <7855100+yantzu@users.noreply.github.com> Date: Tue, 27 Aug 2019 11:37:50 +0800 Subject: [PATCH 3/4] fix checkstyle --- .../src/test/scala/org/apache/livy/sessions/SessionSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala index bd97bc3a4..158603a92 100644 --- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala +++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala @@ -84,7 +84,7 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { Session.prepareConf("", Map(), Nil, Nil, Nil, Seq("file:/not_allowed"), conf) } - //Test owner + // Test owner assert(Session.prepareConf("MyOwner", Map(), Nil, Nil, Nil, Nil, conf) .get("spark.livy.owner") === "MyOwner") assert(Session.prepareConf("", Map(), Nil, Nil, Nil, Nil, conf) From 3cf543480257f428f6d30850ea499e8eb149f85c Mon Sep 17 00:00:00 2001 From: "Jeffrey(Xilang) Yan" <7855100+yantzu@users.noreply.github.com> Date: Tue, 27 Aug 2019 14:41:34 +0800 Subject: [PATCH 4/4] fix ut --- .../test/scala/org/apache/livy/sessions/SessionSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala index 158603a92..bb7946697 100644 --- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala +++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala @@ -86,11 +86,11 @@ class SessionSpec extends FunSuite with LivyBaseUnitTestSuite { // Test owner assert(Session.prepareConf("MyOwner", Map(), Nil, Nil, Nil, Nil, conf) - .get("spark.livy.owner") === "MyOwner") + .get("spark.livy.owner") === Some("MyOwner")) assert(Session.prepareConf("", Map(), Nil, Nil, Nil, Nil, conf) - .get("spark.livy.owner") === "") + .get("spark.livy.owner") === Some("")) assert(Session.prepareConf(null, Map(), Nil, Nil, Nil, Nil, conf) - .get("spark.livy.owner") === "") + .get("spark.livy.owner") === Some("")) // Test that file lists are merged and resolved.