From 682cec48f4abae8d64ade582272fd596d6058725 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Thu, 16 Apr 2020 22:55:02 +0200 Subject: [PATCH 1/6] [LIVY-758] Document how to attach to an existing session from Java client and add a property to the builder --- .../org/apache/livy/LivyClientBuilder.java | 23 +++++++++++++++++++ .../apache/livy/client/http/HttpClient.java | 11 ++++++++- .../livy/client/http/HttpClientSpec.scala | 23 +++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 1d7ec0183..1919cc561 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -36,6 +36,7 @@ public final class LivyClientBuilder { public static final String LIVY_URI_KEY = "livy.uri"; + public static final String LIVY_SESSION_ID_KEY = "livy.sessionId"; private static final ServiceLoader CLIENT_FACTORY_LOADER = ServiceLoader.load(LivyClientFactory.class, classLoader()); @@ -96,11 +97,33 @@ public LivyClientBuilder(boolean loadDefaults) throws IOException { } } + /** + * Set the URI of the Livy server the client will connect to. + * If the URI contains 
sessions/{sessionId}
, + * the client will connect to the specified existing session, + * otherwise it will create a new session. + * + * @param uri The URI of Livy server. + * @return The builder itself. + */ public LivyClientBuilder setURI(URI uri) { config.setProperty(LIVY_URI_KEY, uri.toString()); return this; } + /** + * Sets the sessionId the client will connect to. If sessionId is set, + * all Spark configurations will be ignored and the original session ones will be used. + * If not set, a new session will be created when the client is built. + * + * @param sessionId The ID of the session to attach to. + * @return the builder itself. + */ + public LivyClientBuilder setSessionId(int sessionId) { + config.setProperty(LIVY_SESSION_ID_KEY, String.valueOf(sessionId)); + return this; + } + public LivyClientBuilder setConf(String key, String value) { if (value != null) { config.setProperty(key, value); diff --git a/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java b/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java index f40148f94..82d76d5ee 100644 --- a/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java +++ b/client-http/src/main/java/org/apache/livy/client/http/HttpClient.java @@ -33,6 +33,7 @@ import org.apache.livy.Job; import org.apache.livy.JobHandle; import org.apache.livy.LivyClient; +import org.apache.livy.LivyClientBuilder; import org.apache.livy.client.common.Serializer; import static org.apache.livy.client.common.HttpMessages.*; @@ -59,9 +60,17 @@ class HttpClient implements LivyClient { // unused. Matcher m = Pattern.compile("(.*)" + LivyConnection.SESSIONS_URI + "/([0-9]+)") .matcher(uri.getPath()); + String sessionIdFromConf = httpConf.get(LivyClientBuilder.LIVY_SESSION_ID_KEY); try { - if (m.matches()) { + if (sessionIdFromConf != null && m.matches()) { + throw new IllegalArgumentException( + "Cannot set existing session both from URI and configuration"); + } else if (sessionIdFromConf != null) { + this.conn = new LivyConnection(uri, httpConf); + this.sessionId = Integer.parseInt(sessionIdFromConf); + conn.post(null, SessionInfo.class, "/%d/connect", sessionId); + } else if (m.matches()) { URI base = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), m.group(1), uri.getQuery(), uri.getFragment()); diff --git a/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala b/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala index f53d9f5b4..7ca4f796a 100644 --- a/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala +++ b/client-http/src/test/scala/org/apache/livy/client/http/HttpClientSpec.scala @@ -181,8 +181,8 @@ class HttpClientSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUni testJob(false, response = Some(null)) } - withClient("should connect to existing sessions") { - var sid = client.asInstanceOf[HttpClient].getSessionId() + withClient("should connect to existing sessions using the URI") { + val sid = client.asInstanceOf[HttpClient].getSessionId() val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}" + s"${LivyConnection.SESSIONS_URI}/$sid" val newClient = new LivyClientBuilder(false).setURI(new URI(uri)).build() @@ -190,6 +190,25 @@ class HttpClientSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUni verify(session, never()).stop() } + withClient("should connect to existing sessions using the conf") { + val sid = client.asInstanceOf[HttpClient].getSessionId() + val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}" + val newClient = new LivyClientBuilder(false).setURI(new URI(uri)).setSessionId(sid).build() + newClient.stop(false) + verify(session, never()).stop() + } + + withClient("should throw an exception if the sessionId is set through conf and URI") { + val sid = client.asInstanceOf[HttpClient].getSessionId() + val uri = s"http://${InetAddress.getLocalHost.getHostAddress}:${server.port}" + + s"${LivyConnection.SESSIONS_URI}/$sid" + intercept[IllegalArgumentException]{ + val newClient = new LivyClientBuilder(false).setURI(new URI(uri)).setSessionId(sid).build() + newClient.stop(false) + verify(session, never()).stop() + } + } + withClient("should tear down clients") { client.stop(true) verify(session, times(1)).stop() From 60137e29c67fd3fcb81fe92155e41552b5bcf03a Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Fri, 17 Apr 2020 22:57:14 +0200 Subject: [PATCH 2/6] Update api/src/main/java/org/apache/livy/LivyClientBuilder.java Co-Authored-By: Marco Gaido --- api/src/main/java/org/apache/livy/LivyClientBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 1919cc561..761208bf7 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -100,7 +100,7 @@ public LivyClientBuilder(boolean loadDefaults) throws IOException { /** * Set the URI of the Livy server the client will connect to. * If the URI contains 
sessions/{sessionId}
, - * the client will connect to the specified existing session, + * the client will connect to the specified existing session; * otherwise it will create a new session. * * @param uri The URI of Livy server. From 84a81b42edaee8fd5e63754376e126dbd7ef5466 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Fri, 17 Apr 2020 22:57:26 +0200 Subject: [PATCH 3/6] Update api/src/main/java/org/apache/livy/LivyClientBuilder.java Co-Authored-By: Marco Gaido --- api/src/main/java/org/apache/livy/LivyClientBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 761208bf7..04ea43302 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -113,7 +113,7 @@ public LivyClientBuilder setURI(URI uri) { /** * Sets the sessionId the client will connect to. If sessionId is set, - * all Spark configurations will be ignored and the original session ones will be used. + * the chosen session will be used with its own configurations, so Spark configurations will be ignored. * If not set, a new session will be created when the client is built. * * @param sessionId The ID of the session to attach to. From 04854b2f145d7f409807018124cfe17ddf7b5ae1 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Fri, 17 Apr 2020 22:57:34 +0200 Subject: [PATCH 4/6] Update api/src/main/java/org/apache/livy/LivyClientBuilder.java Co-Authored-By: Marco Gaido --- api/src/main/java/org/apache/livy/LivyClientBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 04ea43302..01049b32a 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -112,7 +112,7 @@ public LivyClientBuilder setURI(URI uri) { } /** - * Sets the sessionId the client will connect to. If sessionId is set, + * Sets the session ID the client will connect to. If a session ID is set, * the chosen session will be used with its own configurations, so Spark configurations will be ignored. * If not set, a new session will be created when the client is built. * From 2ebdbb36703595a354ca237134605259f9447fd2 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Fri, 17 Apr 2020 22:57:44 +0200 Subject: [PATCH 5/6] Update api/src/main/java/org/apache/livy/LivyClientBuilder.java Co-Authored-By: Marco Gaido --- api/src/main/java/org/apache/livy/LivyClientBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index 01049b32a..ebfae280d 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -98,7 +98,7 @@ public LivyClientBuilder(boolean loadDefaults) throws IOException { } /** - * Set the URI of the Livy server the client will connect to. + * Sets the URI of the Livy server the client will connect to. * If the URI contains 
sessions/{sessionId}
, * the client will connect to the specified existing session; * otherwise it will create a new session. From 185b35181f39ca2d49a48773ac9a5ba0f4b074ae Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Mon, 20 Apr 2020 10:41:41 +0200 Subject: [PATCH 6/6] [LIVY-758] Wrap docs to 100 chars --- .../main/java/org/apache/livy/LivyClientBuilder.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java index ebfae280d..654b5502a 100644 --- a/api/src/main/java/org/apache/livy/LivyClientBuilder.java +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -98,9 +98,8 @@ public LivyClientBuilder(boolean loadDefaults) throws IOException { } /** - * Sets the URI of the Livy server the client will connect to. - * If the URI contains 
sessions/{sessionId}
, - * the client will connect to the specified existing session; + * Sets the URI of the Livy server the client will connect to. If the URI contains + * 
sessions/{sessionId}
, the client will connect to the specified existing session; * otherwise it will create a new session. * * @param uri The URI of Livy server. @@ -112,9 +111,9 @@ public LivyClientBuilder setURI(URI uri) { } /** - * Sets the session ID the client will connect to. If a session ID is set, - * the chosen session will be used with its own configurations, so Spark configurations will be ignored. - * If not set, a new session will be created when the client is built. + * Sets the session ID the client will connect to. If a session ID is set, the chosen session + * will be used with its own configurations, so Spark configurations will be ignored. If not set, + * a new session will be created when the client is built. * * @param sessionId The ID of the session to attach to. * @return the builder itself.