From 2765430b0122314cc37270add9510ee4ddfa87be Mon Sep 17 00:00:00 2001 From: David OSullivan Date: Tue, 20 Aug 2019 18:15:12 -0700 Subject: [PATCH 1/5] Added OS utils --- .../client/common/OperatingSystemUtils.java | 81 ++++++++++++ .../common/TestOperatingSystemUtils.java | 123 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java create mode 100644 client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java diff --git a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java new file mode 100644 index 000000000..2d8f2c565 --- /dev/null +++ b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java @@ -0,0 +1,81 @@ +package org.apache.livy.client.common; + +import com.sun.javafx.PlatformUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Collections; +import java.util.EnumSet; +import java.util.function.Supplier; + +import static java.nio.file.attribute.PosixFilePermission.*; + +public class OperatingSystemUtils { + + + @FunctionalInterface + public interface Procedure { + void execute(); + } + + @FunctionalInterface + public interface ThrowingProcedure { + void execute() throws ExceptionType; + } + + @FunctionalInterface + public interface ThrowingSupplier { + T get() throws ExceptionType; + } + + public static boolean isPosixCompliant() { + return PlatformUtil.isMac() || PlatformUtil.isUnix(); + } + + public static boolean isWindows() { + return PlatformUtil.isWindows(); + } + + public static void doBasedOnOs( Procedure doWhenPosixCompliant, Procedure doWhenWindows, String operationDescription) { + doBasedOnOsThrowsException(() -> doWhenPosixCompliant.execute(), () -> doWhenWindows.execute(), operationDescription); + } + + public static void doBasedOnOsThrowsException( ThrowingProcedure doWhenPosixCompliant, ThrowingProcedure doWhenWindows, String operationDescription) throws ExceptionType { + getBasedOnOsThrowsException( + () -> {doWhenPosixCompliant.execute(); return null;}, + () -> {doWhenWindows.execute(); return null;}, + operationDescription); + } + + public static T getBasedOnOs( Supplier doWhenPosixCompliant, Supplier doWhenWindows, String operationDescription) { + return getBasedOnOsThrowsException(() -> doWhenPosixCompliant.get(), () -> doWhenWindows.get(), operationDescription); + } + + public static T getBasedOnOsThrowsException(ThrowingSupplier doWhenPosixCompliant, ThrowingSupplier doWhenWindows, String operationDescription) throws ExceptionType { + if (isPosixCompliant()) { + return doWhenPosixCompliant.get(); + } else if ( isWindows()) { + return doWhenWindows.get(); + } else { + String seperator = operationDescription.isEmpty() ? "" : ": "; + throw new UnsupportedOperationException("Operation" + seperator + operationDescription + " is not supported on this OS"); + } + } + + public static void setOSAgnosticFilePermissions(File file, EnumSet permissions) throws IOException { + OperatingSystemUtils.doBasedOnOsThrowsException( + () -> Files.setPosixFilePermissions(file.toPath(), permissions), + () -> file.setWritable(isWriteable(permissions), permissions.contains(OWNER_WRITE)), //whether or not a file is read-only is the applicable permission on Windows + "Setting file permissions" + ); + } + + private static boolean isWriteable(EnumSet permissions) { + return !Collections.disjoint(permissions, EnumSet.of(OWNER_WRITE, GROUP_WRITE, OTHERS_WRITE)); + } + + + +} diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java new file mode 100644 index 000000000..7eda63c35 --- /dev/null +++ b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java @@ -0,0 +1,123 @@ +package org.apache.livy.client.common; + +import com.sun.javafx.PlatformUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; + +import static java.nio.file.attribute.PosixFilePermission.*; +import static org.junit.Assert.*; + +public class TestOperatingSystemUtils { + + @Before + public void setUp() throws UnsupportedOperationException { + if ( !(PlatformUtil.isMac() || PlatformUtil.isWindows() || PlatformUtil.isUnix())) { + throw new UnsupportedOperationException("Not running on supported OS"); + } + } + + @Test + public void testIsPosixCompliant() { + assertEquals(PlatformUtil.isUnix() || PlatformUtil.isMac(), OperatingSystemUtils.isPosixCompliant()); + } + + @Test + public void testIsWindows() { + assertEquals(PlatformUtil.isWindows(), OperatingSystemUtils.isWindows()); + } + + @Test + public void testDoBasedOnOs() { + if (OperatingSystemUtils.isPosixCompliant()) { + OperatingSystemUtils.doBasedOnOs( + () -> {}, + () -> fail(), + "" + ); + } else if (OperatingSystemUtils.isWindows()) { + OperatingSystemUtils.doBasedOnOs( + () -> fail(), + () -> {}, + "" + ); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testDoBasedOnOsThrowsException() { + if (OperatingSystemUtils.isPosixCompliant()) { + OperatingSystemUtils.doBasedOnOsThrowsException( + () -> {throw new IllegalArgumentException();}, + () -> fail(), + "" + ); + } else if (OperatingSystemUtils.isWindows()) { + OperatingSystemUtils.doBasedOnOsThrowsException( + () -> fail(), + () -> {throw new IllegalArgumentException();}, + "" + ); + } + } + + @Test + public void testGetBasedOnOs() { + String value = ""; + if (OperatingSystemUtils.isPosixCompliant()) { + value = OperatingSystemUtils.getBasedOnOs( + () -> "pass", + () -> "fail", + "" + ); + } else if (OperatingSystemUtils.isWindows()) { + value = OperatingSystemUtils.getBasedOnOs( + () -> "fail", + () -> "pass", + "" + ); + } + assertEquals(value, "pass"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetBasedOnOsThrowsException() { + if (OperatingSystemUtils.isPosixCompliant()) { + OperatingSystemUtils.getBasedOnOsThrowsException( + () -> {throw new IllegalArgumentException();}, + () -> "fail", + "" + ); + } else if (OperatingSystemUtils.isWindows()) { + OperatingSystemUtils.getBasedOnOsThrowsException( + () -> "fail", + () -> {throw new IllegalArgumentException();}, + "" + ); + } + fail(); + } + + @Test + public void testSetOSAgnosticFilePermissions() throws IOException { + File f = File.createTempFile("testFile", ".txt"); + f.deleteOnExit(); + OperatingSystemUtils.setOSAgnosticFilePermissions(f, EnumSet.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)); + assertTrue(f.canExecute()); + assertTrue(f.canWrite()); + assertTrue(f.canRead()); + + OperatingSystemUtils.setOSAgnosticFilePermissions(f, EnumSet.of(OWNER_READ)); + if (OperatingSystemUtils.isPosixCompliant()) { + assertFalse(f.canExecute()); + } + assertFalse(f.canWrite()); + assertTrue(f.canRead()); + + + } +} \ No newline at end of file From 557f6c49909bdb91383e0ff148bbb51094e97949 Mon Sep 17 00:00:00 2001 From: David OSullivan Date: Tue, 20 Aug 2019 18:16:57 -0700 Subject: [PATCH 2/5] Changed OS dependent calls in LivyConf, ContextLauncher, RSCDriver --- .../org/apache/livy/rsc/ContextLauncher.java | 4 ++-- .../org/apache/livy/rsc/driver/RSCDriver.java | 17 +++++++---------- .../main/scala/org/apache/livy/LivyConf.scala | 10 ++++++---- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java index 5a819d563..1381fbd79 100644 --- a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java +++ b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java @@ -44,6 +44,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.util.concurrent.Promise; +import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.spark.launcher.SparkLauncher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -299,9 +300,8 @@ private static File writeConfToFile(RSCConf conf) throws IOException { } } } - File file = File.createTempFile("livyConf", ".properties"); - Files.setPosixFilePermissions(file.toPath(), EnumSet.of(OWNER_READ, OWNER_WRITE)); + OperatingSystemUtils.setOSAgnosticFilePermissions(file, EnumSet.of(OWNER_READ, OWNER_WRITE)); //file.deleteOnExit(); Writer writer = new OutputStreamWriter(new FileOutputStream(file), UTF_8); diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java index eeba3001d..7bf230b8c 100644 --- a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java +++ b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java @@ -24,13 +24,7 @@ import java.nio.file.Files; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutorService; @@ -47,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.slf4j.Logger; @@ -62,6 +57,7 @@ import org.apache.livy.rsc.rpc.RpcDispatcher; import org.apache.livy.rsc.rpc.RpcServer; +import static java.nio.file.attribute.PosixFilePermission.*; import static org.apache.livy.rsc.RSCConf.Entry.*; /** @@ -96,9 +92,10 @@ public class RSCDriver extends BaseProtocol { private final AtomicBoolean inShutdown; public RSCDriver(SparkConf conf, RSCConf livyConf) throws Exception { - Set perms = PosixFilePermissions.fromString("rwx------"); - this.localTmpDir = Files.createTempDirectory("rsc-tmp", - PosixFilePermissions.asFileAttribute(perms)).toFile(); + + this.localTmpDir = Files.createTempDirectory("rsc-tmp").toFile(); + OperatingSystemUtils.setOSAgnosticFilePermissions(this.localTmpDir, EnumSet.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)); + this.executor = Executors.newCachedThreadPool(); this.jobQueue = new LinkedList<>(); this.clients = new ConcurrentLinkedDeque<>(); diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala index dec8e4aff..bd8857fa1 100644 --- a/server/src/main/scala/org/apache/livy/LivyConf.scala +++ b/server/src/main/scala/org/apache/livy/LivyConf.scala @@ -19,13 +19,12 @@ package org.apache.livy import java.io.File import java.lang.{Boolean => JBoolean, Long => JLong} +import java.util.function.{Consumer, Supplier} import java.util.{Map => JMap} import scala.collection.JavaConverters._ - import org.apache.hadoop.conf.Configuration - -import org.apache.livy.client.common.ClientConf +import org.apache.livy.client.common.{ClientConf, OperatingSystemUtils} import org.apache.livy.client.common.ClientConf.ConfEntry import org.apache.livy.client.common.ClientConf.DeprecatedConf @@ -335,7 +334,10 @@ class LivyConf(loadDefaults: Boolean) extends ClientConf[LivyConf](null) { /** Return the path to the spark-submit executable. */ def sparkSubmit(): String = { - sparkHome().map { _ + File.separator + "bin" + File.separator + "spark-submit" }.get + val posixSupplier = new Supplier[String]() { override def get() = "spark-submit" }; + val windowsSupplier = new Supplier[String]() { override def get() = "spark-submit.cmd" }; + val sparkSubmit = OperatingSystemUtils.getBasedOnOs(posixSupplier, windowsSupplier, "Getting Spark Submit"); + sparkHome().map { _ + File.separator + "bin" + File.separator + sparkSubmit }.get } private val configDir: Option[File] = { From 87a3873933f43940a817e4e2d7a80603041de51c Mon Sep 17 00:00:00 2001 From: David OSullivan Date: Wed, 21 Aug 2019 15:36:50 -0700 Subject: [PATCH 3/5] refactoring --- .../client/common/OperatingSystemUtils.java | 41 ++++++++-------- .../common/TestOperatingSystemUtils.java | 47 +++++++++---------- .../main/scala/org/apache/livy/LivyConf.scala | 11 +++-- 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java index 2d8f2c565..c0eba4d1e 100644 --- a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java +++ b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.livy.client.common; import com.sun.javafx.PlatformUtil; @@ -8,13 +25,11 @@ import java.nio.file.attribute.PosixFilePermission; import java.util.Collections; import java.util.EnumSet; -import java.util.function.Supplier; import static java.nio.file.attribute.PosixFilePermission.*; public class OperatingSystemUtils { - @FunctionalInterface public interface Procedure { void execute(); @@ -25,11 +40,6 @@ public interface ThrowingProcedure { void execute() throws ExceptionType; } - @FunctionalInterface - public interface ThrowingSupplier { - T get() throws ExceptionType; - } - public static boolean isPosixCompliant() { return PlatformUtil.isMac() || PlatformUtil.isUnix(); } @@ -43,21 +53,14 @@ public static void doBasedOnOs( Procedure doWhenPosixCompliant, Procedure doWhe } public static void doBasedOnOsThrowsException( ThrowingProcedure doWhenPosixCompliant, ThrowingProcedure doWhenWindows, String operationDescription) throws ExceptionType { - getBasedOnOsThrowsException( - () -> {doWhenPosixCompliant.execute(); return null;}, - () -> {doWhenWindows.execute(); return null;}, - operationDescription); - } - - public static T getBasedOnOs( Supplier doWhenPosixCompliant, Supplier doWhenWindows, String operationDescription) { - return getBasedOnOsThrowsException(() -> doWhenPosixCompliant.get(), () -> doWhenWindows.get(), operationDescription); + getBasedOnOS(doWhenPosixCompliant, doWhenWindows, operationDescription).execute(); } - public static T getBasedOnOsThrowsException(ThrowingSupplier doWhenPosixCompliant, ThrowingSupplier doWhenWindows, String operationDescription) throws ExceptionType { + public static T getBasedOnOS(T getWhenPosixCompliant, T getWhenWindows, String operationDescription) { if (isPosixCompliant()) { - return doWhenPosixCompliant.get(); + return getWhenPosixCompliant; } else if ( isWindows()) { - return doWhenWindows.get(); + return getWhenWindows; } else { String seperator = operationDescription.isEmpty() ? "" : ": "; throw new UnsupportedOperationException("Operation" + seperator + operationDescription + " is not supported on this OS"); @@ -67,7 +70,7 @@ public static T getBasedOnOsThrowsException public static void setOSAgnosticFilePermissions(File file, EnumSet permissions) throws IOException { OperatingSystemUtils.doBasedOnOsThrowsException( () -> Files.setPosixFilePermissions(file.toPath(), permissions), - () -> file.setWritable(isWriteable(permissions), permissions.contains(OWNER_WRITE)), //whether or not a file is read-only is the applicable permission on Windows + () -> file.setWritable(isWriteable(permissions), permissions.contains(OWNER_WRITE)), //whether or not a file is read-only is the only applicable permission on Windows "Setting file permissions" ); } diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java index 7eda63c35..650afd621 100644 --- a/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java +++ b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.livy.client.common; import com.sun.javafx.PlatformUtil; @@ -69,39 +86,21 @@ public void testDoBasedOnOsThrowsException() { public void testGetBasedOnOs() { String value = ""; if (OperatingSystemUtils.isPosixCompliant()) { - value = OperatingSystemUtils.getBasedOnOs( - () -> "pass", - () -> "fail", + value = OperatingSystemUtils.getBasedOnOS( + "pass", + "fail", "" ); } else if (OperatingSystemUtils.isWindows()) { - value = OperatingSystemUtils.getBasedOnOs( - () -> "fail", - () -> "pass", + value = OperatingSystemUtils.getBasedOnOS( + "fail", + "pass", "" ); } assertEquals(value, "pass"); } - @Test(expected = IllegalArgumentException.class) - public void testGetBasedOnOsThrowsException() { - if (OperatingSystemUtils.isPosixCompliant()) { - OperatingSystemUtils.getBasedOnOsThrowsException( - () -> {throw new IllegalArgumentException();}, - () -> "fail", - "" - ); - } else if (OperatingSystemUtils.isWindows()) { - OperatingSystemUtils.getBasedOnOsThrowsException( - () -> "fail", - () -> {throw new IllegalArgumentException();}, - "" - ); - } - fail(); - } - @Test public void testSetOSAgnosticFilePermissions() throws IOException { File f = File.createTempFile("testFile", ".txt"); diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala b/server/src/main/scala/org/apache/livy/LivyConf.scala index bd8857fa1..a0f719435 100644 --- a/server/src/main/scala/org/apache/livy/LivyConf.scala +++ b/server/src/main/scala/org/apache/livy/LivyConf.scala @@ -19,11 +19,12 @@ package org.apache.livy import java.io.File import java.lang.{Boolean => JBoolean, Long => JLong} -import java.util.function.{Consumer, Supplier} import java.util.{Map => JMap} import scala.collection.JavaConverters._ + import org.apache.hadoop.conf.Configuration + import org.apache.livy.client.common.{ClientConf, OperatingSystemUtils} import org.apache.livy.client.common.ClientConf.ConfEntry import org.apache.livy.client.common.ClientConf.DeprecatedConf @@ -334,9 +335,11 @@ class LivyConf(loadDefaults: Boolean) extends ClientConf[LivyConf](null) { /** Return the path to the spark-submit executable. */ def sparkSubmit(): String = { - val posixSupplier = new Supplier[String]() { override def get() = "spark-submit" }; - val windowsSupplier = new Supplier[String]() { override def get() = "spark-submit.cmd" }; - val sparkSubmit = OperatingSystemUtils.getBasedOnOs(posixSupplier, windowsSupplier, "Getting Spark Submit"); + val sparkSubmit = OperatingSystemUtils + .getBasedOnOS( + "spark-submit", + "spark-submit.cmd", + "Getting Spark Submit"); sparkHome().map { _ + File.separator + "bin" + File.separator + sparkSubmit }.get } From 5eae63c1eb3804694f5b8f0cbb94199ea72d1cb9 Mon Sep 17 00:00:00 2001 From: David OSullivan Date: Wed, 21 Aug 2019 21:21:36 -0700 Subject: [PATCH 4/5] refactoring for checkstyle --- .../client/common/OperatingSystemUtils.java | 63 ++++++++++++------- .../common/TestOperatingSystemUtils.java | 21 ++++--- .../org/apache/livy/rsc/ContextLauncher.java | 3 +- .../org/apache/livy/rsc/driver/RSCDriver.java | 12 ++-- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java index c0eba4d1e..bf0142d4f 100644 --- a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java +++ b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java @@ -17,7 +17,6 @@ package org.apache.livy.client.common; -import com.sun.javafx.PlatformUtil; import java.io.File; import java.io.IOException; @@ -25,20 +24,14 @@ import java.nio.file.attribute.PosixFilePermission; import java.util.Collections; import java.util.EnumSet; - import static java.nio.file.attribute.PosixFilePermission.*; -public class OperatingSystemUtils { - - @FunctionalInterface - public interface Procedure { - void execute(); - } +import com.sun.javafx.PlatformUtil; - @FunctionalInterface - public interface ThrowingProcedure { - void execute() throws ExceptionType; - } +/* + * Utility class to get OS related information and perform OS-dependent actions + */ +public class OperatingSystemUtils { public static boolean isPosixCompliant() { return PlatformUtil.isMac() || PlatformUtil.isUnix(); @@ -48,37 +41,63 @@ public static boolean isWindows() { return PlatformUtil.isWindows(); } - public static void doBasedOnOs( Procedure doWhenPosixCompliant, Procedure doWhenWindows, String operationDescription) { - doBasedOnOsThrowsException(() -> doWhenPosixCompliant.execute(), () -> doWhenWindows.execute(), operationDescription); + public static void doBasedOnOs( + Procedure doWhenPosixCompliant, + Procedure doWhenWindows, + String operationDescription) { + doBasedOnOsThrowsException( + () -> doWhenPosixCompliant.execute(), + () -> doWhenWindows.execute(), + operationDescription); } - public static void doBasedOnOsThrowsException( ThrowingProcedure doWhenPosixCompliant, ThrowingProcedure doWhenWindows, String operationDescription) throws ExceptionType { + public static void doBasedOnOsThrowsException( + ThrowingProcedure doWhenPosixCompliant, + ThrowingProcedure doWhenWindows, + String operationDescription) throws ExceptionType { getBasedOnOS(doWhenPosixCompliant, doWhenWindows, operationDescription).execute(); } - public static T getBasedOnOS(T getWhenPosixCompliant, T getWhenWindows, String operationDescription) { + public static T getBasedOnOS( + T getWhenPosixCompliant, + T getWhenWindows, + String operationDescription) { if (isPosixCompliant()) { return getWhenPosixCompliant; - } else if ( isWindows()) { + } else if (isWindows()) { return getWhenWindows; } else { String seperator = operationDescription.isEmpty() ? "" : ": "; - throw new UnsupportedOperationException("Operation" + seperator + operationDescription + " is not supported on this OS"); + throw new UnsupportedOperationException("Operation" + seperator + operationDescription + + " is not supported on this OS"); } } - public static void setOSAgnosticFilePermissions(File file, EnumSet permissions) throws IOException { + public static void setOSAgnosticFilePermissions(File file, + EnumSet permissions) + throws IOException { OperatingSystemUtils.doBasedOnOsThrowsException( () -> Files.setPosixFilePermissions(file.toPath(), permissions), - () -> file.setWritable(isWriteable(permissions), permissions.contains(OWNER_WRITE)), //whether or not a file is read-only is the only applicable permission on Windows + //whether or not a file is read-only is the only applicable permission on Windows + () -> file.setWritable(isWriteable(permissions), permissions.contains(OWNER_WRITE)), "Setting file permissions" - ); + ); } private static boolean isWriteable(EnumSet permissions) { - return !Collections.disjoint(permissions, EnumSet.of(OWNER_WRITE, GROUP_WRITE, OTHERS_WRITE)); + return !Collections.disjoint(permissions, EnumSet.of(OWNER_WRITE, GROUP_WRITE, + OTHERS_WRITE)); + } + + @FunctionalInterface + public interface Procedure { + void execute(); } + @FunctionalInterface + public interface ThrowingProcedure { + void execute() throws ExceptionType; + } } diff --git a/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java index 650afd621..47b3126b1 100644 --- a/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java +++ b/client-common/src/test/java/org/apache/livy/client/common/TestOperatingSystemUtils.java @@ -17,18 +17,17 @@ package org.apache.livy.client.common; -import com.sun.javafx.PlatformUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - import java.io.File; import java.io.IOException; import java.util.EnumSet; - import static java.nio.file.attribute.PosixFilePermission.*; + +import com.sun.javafx.PlatformUtil; +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.*; + public class TestOperatingSystemUtils { @Before @@ -40,12 +39,14 @@ public void setUp() throws UnsupportedOperationException { @Test public void testIsPosixCompliant() { - assertEquals(PlatformUtil.isUnix() || PlatformUtil.isMac(), OperatingSystemUtils.isPosixCompliant()); + assertEquals(PlatformUtil.isUnix() || PlatformUtil.isMac(), + OperatingSystemUtils.isPosixCompliant()); } @Test public void testIsWindows() { - assertEquals(PlatformUtil.isWindows(), OperatingSystemUtils.isWindows()); + assertEquals(PlatformUtil.isWindows(), + OperatingSystemUtils.isWindows()); } @Test @@ -105,7 +106,9 @@ public void testGetBasedOnOs() { public void testSetOSAgnosticFilePermissions() throws IOException { File f = File.createTempFile("testFile", ".txt"); f.deleteOnExit(); - OperatingSystemUtils.setOSAgnosticFilePermissions(f, EnumSet.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)); + OperatingSystemUtils.setOSAgnosticFilePermissions(f, EnumSet.of(OWNER_READ, + OWNER_WRITE, + OWNER_EXECUTE)); assertTrue(f.canExecute()); assertTrue(f.canWrite()); assertTrue(f.canRead()); diff --git a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java index 1381fbd79..518ba8034 100644 --- a/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java +++ b/rsc/src/main/java/org/apache/livy/rsc/ContextLauncher.java @@ -44,17 +44,16 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.util.concurrent.Promise; -import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.spark.launcher.SparkLauncher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.livy.client.common.TestUtils; import org.apache.livy.rsc.driver.RSCDriverBootstrapper; import org.apache.livy.rsc.rpc.Rpc; import org.apache.livy.rsc.rpc.RpcDispatcher; import org.apache.livy.rsc.rpc.RpcServer; - import static org.apache.livy.rsc.RSCConf.Entry.*; /** diff --git a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java index 7bf230b8c..782eeb6ef 100644 --- a/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java +++ b/rsc/src/main/java/org/apache/livy/rsc/driver/RSCDriver.java @@ -17,13 +17,12 @@ package org.apache.livy.rsc.driver; + import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.nio.file.Files; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; @@ -33,6 +32,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static java.nio.file.attribute.PosixFilePermission.*; import io.netty.channel.ChannelHandler.Sharable; import io.netty.channel.ChannelHandlerContext; @@ -41,12 +41,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.livy.client.common.OperatingSystemUtils; import org.apache.livy.client.common.Serializer; import org.apache.livy.rsc.BaseProtocol; import org.apache.livy.rsc.BypassJobStatus; @@ -56,8 +56,6 @@ import org.apache.livy.rsc.rpc.Rpc; import org.apache.livy.rsc.rpc.RpcDispatcher; import org.apache.livy.rsc.rpc.RpcServer; - -import static java.nio.file.attribute.PosixFilePermission.*; import static org.apache.livy.rsc.RSCConf.Entry.*; /** @@ -94,7 +92,9 @@ public class RSCDriver extends BaseProtocol { public RSCDriver(SparkConf conf, RSCConf livyConf) throws Exception { this.localTmpDir = Files.createTempDirectory("rsc-tmp").toFile(); - OperatingSystemUtils.setOSAgnosticFilePermissions(this.localTmpDir, EnumSet.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)); + OperatingSystemUtils.setOSAgnosticFilePermissions(this.localTmpDir, EnumSet.of(OWNER_READ, + OWNER_WRITE, + OWNER_EXECUTE)); this.executor = Executors.newCachedThreadPool(); this.jobQueue = new LinkedList<>(); From fe20e171be4ddbd2d8a0b1f0fd3bde53242ea156 Mon Sep 17 00:00:00 2001 From: David OSullivan Date: Wed, 21 Aug 2019 21:29:01 -0700 Subject: [PATCH 5/5] new line --- .../org/apache/livy/client/common/OperatingSystemUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java index bf0142d4f..cf89e2f4e 100644 --- a/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java +++ b/client-common/src/main/java/org/apache/livy/client/common/OperatingSystemUtils.java @@ -85,7 +85,8 @@ public static void setOSAgnosticFilePermissions(File file, } private static boolean isWriteable(EnumSet permissions) { - return !Collections.disjoint(permissions, EnumSet.of(OWNER_WRITE, GROUP_WRITE, + return !Collections.disjoint(permissions, EnumSet.of(OWNER_WRITE, + GROUP_WRITE, OTHERS_WRITE)); }