diff --git a/api/src/main/java/org/apache/livy/JobContext.java b/api/src/main/java/org/apache/livy/JobContext.java index 7baab9a01..605d10077 100644 --- a/api/src/main/java/org/apache/livy/JobContext.java +++ b/api/src/main/java/org/apache/livy/JobContext.java @@ -27,7 +27,7 @@ /** * Holds runtime information about the job execution context. - * + *
* An instance of this class is kept on the node hosting a remote Spark context and is made * available to jobs being executed via RemoteSparkContext#submit(). */ diff --git a/api/src/main/java/org/apache/livy/JobHandle.java b/api/src/main/java/org/apache/livy/JobHandle.java index 75f7a59f0..ba8511813 100644 --- a/api/src/main/java/org/apache/livy/JobHandle.java +++ b/api/src/main/java/org/apache/livy/JobHandle.java @@ -43,7 +43,7 @@ public interface JobHandle extends Future { /** * The current state of the submitted job. */ - static enum State { + enum State { SENT, QUEUED, STARTED, @@ -56,7 +56,7 @@ static enum State { * A listener for monitoring the state of the job in the remote context. Callbacks are called * when the corresponding state change occurs. */ - static interface Listener { + interface Listener { /** * Notifies when a job has been queued for execution on the remote context. Note that it is diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 1d7ec0183..b7897dcd1 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -65,7 +65,7 @@ public LivyClientBuilder() throws IOException { /** * Creates a new builder that will optionally load the default Livy and Spark configuration * from the classpath. - * + *
* Livy client configuration is stored in a file called "livy-client.conf", and Spark client * configuration is stored in a file called "spark-defaults.conf", both in the root of the * application's classpath. Livy configuration takes precedence over Spark's (in case @@ -85,12 +85,9 @@ public LivyClientBuilder(boolean loadDefaults) throws IOException { for (String file : confFiles) { URL url = classLoader().getResource(file); if (url != null) { - Reader r = new InputStreamReader(url.openStream(), UTF_8); - try { - config.load(r); - } finally { - r.close(); - } + try (Reader r = new InputStreamReader(url.openStream(), UTF_8)) { + config.load(r); + } } } } @@ -123,7 +120,7 @@ public LivyClientBuilder setAll(Properties props) { public LivyClient build() { String uriStr = config.getProperty(LIVY_URI_KEY); if (uriStr == null) { - throw new IllegalArgumentException("URI must be provided."); + throw new IllegalArgumentException("URI must be provided in " + LIVY_URI_KEY + "."); } URI uri; try { @@ -138,14 +135,7 @@ public LivyClient build() { } for (LivyClientFactory factory : CLIENT_FACTORIES) { - try { - client = factory.createClient(uri, config); - } catch (Exception e) { - if (!(e instanceof RuntimeException)) { - e = new RuntimeException(e); - } - throw (RuntimeException) e; - } + client = factory.createClient(uri, config); if (client != null) { break; } diff --git a/api/src/test/java/org/apache/livy/TestClientFactory.java b/api/src/test/java/org/apache/livy/TestClientFactory.java index 622908c04..9c07096f9 100644 --- a/api/src/test/java/org/apache/livy/TestClientFactory.java +++ b/api/src/test/java/org/apache/livy/TestClientFactory.java @@ -25,7 +25,7 @@ public class TestClientFactory implements LivyClientFactory { - private static AtomicLong instanceCount = new AtomicLong(); + private static final AtomicLong instanceCount = new AtomicLong(); public static long getInstanceCount() { return instanceCount.get(); } diff --git a/core/pom.xml b/core/pom.xml index fca0600b7..363536eb5 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -68,4 +68,52 @@ + + + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.version} + + + + compile + testCompile + doc-jar + + + + scala-compile-first + process-resources + + compile + + + + + ${scala.version} + incremental + false + + -unchecked + -deprecation + -feature + + + -source + ${java.version} + -target + ${java.version} + -Xlint:unchecked + + + -Xms1024m + -Xmx1024m + -XX:ReservedCodeCacheSize=${CodeCacheSize} + + + + + + diff --git a/integration-test/pom.xml b/integration-test/pom.xml index a803b3573..ce7dcc438 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -290,6 +290,51 @@ ${skipITs} + + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.version} + + + + compile + testCompile + doc-jar + + + + scala-compile-first + process-resources + + compile + + + + + ${scala.version} + incremental + false + + -unchecked + -deprecation + -feature + + + -source + ${java.version} + -target + ${java.version} + -Xlint:unchecked + + + -Xms1024m + -Xmx1024m + -XX:ReservedCodeCacheSize=${CodeCacheSize} + + + + diff --git a/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala b/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala index e0fedfb20..750d97421 100644 --- a/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala +++ b/integration-test/src/main/scala/org/apache/livy/test/framework/LivyRestClient.scala @@ -82,8 +82,8 @@ class LivyRestClient(val httpClient: CloseableHttpClient, val livyEndpoint: Stri def snapshot(): SessionSnapshot = { val httpGet = new HttpGet(url) val r = httpClient.execute(httpGet) - val statusLine = r.getStatusLine() - val responseBody = r.getEntity().getContent + val statusLine = r.getStatusLine + val responseBody = r.getEntity.getContent val sessionSnapshot = mapper.readValue(responseBody, classOf[SessionSnapshot]) r.close() diff --git a/pom.xml b/pom.xml index 4dc328e96..ef35ad9af 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,7 @@ 3.0.8 2.6.5 1.8 + 4.3.0 -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang.invoke=ALL-UNNAMED @@ -907,7 +908,7 @@ net.alchim31.maven scala-maven-plugin - 4.3.0 + ${scala.maven.version} diff --git a/repl/pom.xml b/repl/pom.xml index b3e96324f..fbbb9349e 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -262,6 +262,7 @@ + diff --git a/repl/scala-2.12/pom.xml b/repl/scala-2.12/pom.xml index 9a1e87d99..c7b322c52 100644 --- a/repl/scala-2.12/pom.xml +++ b/repl/scala-2.12/pom.xml @@ -31,4 +31,52 @@ ../pom.xml + + + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.version} + + + + compile + testCompile + doc-jar + + + + scala-compile-first + process-resources + + compile + + + + + ${scala.version} + incremental + false + + -unchecked + -deprecation + -feature + + + -source + ${java.version} + -target + ${java.version} + -Xlint:unchecked + + + -Xms1024m + -Xmx1024m + -XX:ReservedCodeCacheSize=${CodeCacheSize} + + + + + + diff --git a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java index 6bf8f6ea7..e0405de76 100644 --- a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java +++ b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java @@ -23,6 +23,8 @@ import java.io.PipedOutputStream; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.time.Duration; import java.util.*; import java.util.concurrent.*; @@ -231,9 +233,9 @@ public void call(LivyClient client) throws Exception { // Test that adding a jar to the remote context makes it show up in the classpath. jar = File.createTempFile("test", ".jar"); - JarOutputStream jarFile = new JarOutputStream(new FileOutputStream(jar)); + JarOutputStream jarFile = new JarOutputStream(Files.newOutputStream(jar.toPath())); jarFile.putNextEntry(new ZipEntry("test.resource")); - jarFile.write("test resource".getBytes("UTF-8")); + jarFile.write("test resource".getBytes(StandardCharsets.UTF_8)); jarFile.closeEntry(); jarFile.close(); @@ -256,7 +258,7 @@ public void call(LivyClient client) throws Exception { file = File.createTempFile("test", ".file"); FileOutputStream fileStream = new FileOutputStream(file); - fileStream.write("test file".getBytes("UTF-8")); + fileStream.write("test file".getBytes(StandardCharsets.UTF_8)); fileStream.close(); client.addJar(new URI("file:" + file.getAbsolutePath())) @@ -403,54 +405,52 @@ void config(Properties conf) { public void testKillServerWhileSparkSubmitIsRunning() throws Exception { Properties conf = createConf(true); LivyClient client = null; - PipedInputStream stubStream = new PipedInputStream(new PipedOutputStream()); - try { - Process mockSparkSubmit = mock(Process.class); - when(mockSparkSubmit.getInputStream()).thenReturn(stubStream); - when(mockSparkSubmit.getErrorStream()).thenReturn(stubStream); - - // Block waitFor until process.destroy() is called. - final CountDownLatch waitForCalled = new CountDownLatch(1); - when(mockSparkSubmit.waitFor()).thenAnswer(new Answer() { - @Override - public Integer answer(InvocationOnMock invocation) throws Throwable { - waitForCalled.await(); - return 0; - } - }); - - // Verify process.destroy() is called. - final CountDownLatch destroyCalled = new CountDownLatch(1); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - destroyCalled.countDown(); - return null; - } - }).when(mockSparkSubmit).destroy(); - - ContextLauncher.mockSparkSubmit = mockSparkSubmit; - - client = new LivyClientBuilder(false).setURI(new URI("rsc:/")) - .setAll(conf) - .build(); - - client.stop(true); - - assertTrue(destroyCalled.await(5, TimeUnit.SECONDS)); - waitForCalled.countDown(); - } catch (Exception e) { - // JUnit prints not so useful backtraces in test summary reports, and we don't see the - // actual source line of the exception, so print the exception to the logs. - LOG.error("Test threw exception.", e); - throw e; - } finally { - ContextLauncher.mockSparkSubmit = null; - stubStream.close(); - if (client != null) { - client.stop(true); + try (PipedInputStream stubStream = new PipedInputStream(new PipedOutputStream())) { + Process mockSparkSubmit = mock(Process.class); + when(mockSparkSubmit.getInputStream()).thenReturn(stubStream); + when(mockSparkSubmit.getErrorStream()).thenReturn(stubStream); + + // Block waitFor until process.destroy() is called. + final CountDownLatch waitForCalled = new CountDownLatch(1); + when(mockSparkSubmit.waitFor()).thenAnswer(new Answer() { + @Override + public Integer answer(InvocationOnMock invocation) throws Throwable { + waitForCalled.await(); + return 0; + } + }); + + // Verify process.destroy() is called. + final CountDownLatch destroyCalled = new CountDownLatch(1); + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + destroyCalled.countDown(); + return null; + } + }).when(mockSparkSubmit).destroy(); + + ContextLauncher.mockSparkSubmit = mockSparkSubmit; + + client = new LivyClientBuilder(false).setURI(new URI("rsc:/")) + .setAll(conf) + .build(); + + client.stop(true); + + assertTrue(destroyCalled.await(5, TimeUnit.SECONDS)); + waitForCalled.countDown(); + } catch (Exception e) { + // JUnit prints not so useful backtraces in test summary reports, and we don't see the + // actual source line of the exception, so print the exception to the logs. + LOG.error("Test threw exception.", e); + throw e; + } finally { + ContextLauncher.mockSparkSubmit = null; + if (client != null) { + client.stop(true); + } } - } } @Test diff --git a/scala-api/pom.xml b/scala-api/pom.xml index 991fb4412..c18dad828 100644 --- a/scala-api/pom.xml +++ b/scala-api/pom.xml @@ -94,5 +94,53 @@ provided + + + + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.version} + + + + compile + testCompile + doc-jar + + + + scala-compile-first + process-resources + + compile + + + + + ${scala.version} + incremental + false + + -unchecked + -deprecation + -feature + + + -source + ${java.version} + -target + ${java.version} + -Xlint:unchecked + + + -Xms1024m + -Xmx1024m + -XX:ReservedCodeCacheSize=${CodeCacheSize} + + + + + diff --git a/server/pom.xml b/server/pom.xml index 29fe3555c..1e768a13b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -402,6 +402,49 @@ + + net.alchim31.maven + scala-maven-plugin + ${scala.maven.version} + + + + compile + testCompile + doc-jar + + + + scala-compile-first + process-resources + + compile + + + + + ${scala.version} + incremental + false + + -unchecked + -deprecation + -feature + + + -source + ${java.version} + -target + ${java.version} + -Xlint:unchecked + + + -Xms1024m + -Xmx1024m + -XX:ReservedCodeCacheSize=${CodeCacheSize} + + +