From 91b16de620cc7f3168ca3e613c22c3f9c777fe97 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 09:37:45 -0400 Subject: [PATCH 01/57] Add debug code --- .../java/com/github/ibm/mapepire/DataStreamProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 462f2be..fe32592 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -135,6 +135,12 @@ public void run(String requestString) { m_prepStmtMap.remove(cont_id.getAsString()); break; } +// case "blob": { +// final RunBlob blob = new RunBlob(this, m_conn, reqObj); +// m_queriesMap.put(runSqlReq.getId(), runSqlReq); +// dispatch(runSqlReq); +// break; +// } case "execute": if (null == cont_id) { dispatch(new BadReq(this, m_conn, reqObj, "Correlation ID not specified")); From 953eed795209c32c16946beefaf155c5d1a5c0b6 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 12:27:07 -0400 Subject: [PATCH 02/57] add support for all types --- .../mapepire/requests/PreparedExecute.java | 102 ++++++++++++++++-- 1 file changed, 95 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java index 0f7a6a4..fa2d321 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java @@ -1,11 +1,20 @@ package com.github.ibm.mapepire.requests; +import java.io.ByteArrayInputStream; +import java.io.StringReader; +import java.math.BigDecimal; import java.sql.CallableStatement; import java.sql.ParameterMetaData; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; + + import java.sql.Types; import java.util.Arrays; +import java.util.Base64; import java.util.LinkedList; import com.github.ibm.mapepire.DataStreamProcessor; @@ -25,6 +34,7 @@ public PreparedExecute(final DataStreamProcessor _io, final JsonObject _reqObj, @Override protected void go() throws Exception { JsonArray parms = super.getRequestField("parameters").getAsJsonArray(); + JsonArray columnTypes = super.getRequestField("columnTypes").getAsJsonArray(); boolean isBatch = !parms.isEmpty() && parms.get(0).isJsonArray(); boolean hasResultSet = false; long batchUpdateCount = 0; @@ -34,7 +44,7 @@ protected void go() throws Exception { JsonArray arr = parms.getAsJsonArray(); int batch_ops_added = 0; for (int i = 0; i < arr.size(); i++) { - addJsonArrayParameters(stmt, arr.get(i).getAsJsonArray()); + addJsonArrayParameters(stmt, arr.get(i).getAsJsonArray(), columnTypes); m_prev.getStatement().addBatch(); } batch_ops_added += arr.size(); @@ -43,7 +53,7 @@ protected void go() throws Exception { batchUpdateCount = Arrays.stream(updateCount).sum(); } else{ if (parms != null) { - addJsonArrayParameters(stmt, parms.getAsJsonArray()); + addJsonArrayParameters(stmt, parms.getAsJsonArray(), columnTypes); } hasResultSet = stmt.execute(); } @@ -66,9 +76,87 @@ protected void go() throws Exception { } } - private void addJsonArrayParameters(PreparedStatement stmt, JsonArray arr) throws SQLException { - for (int i = 1; i <= arr.size(); ++i) { - JsonElement element = arr.get(-1 + i); + private void addParameter(int i, JsonElement parameter, ColumnType columnType, PreparedStatement stmt) throws SQLException { + String stringValue = parameter.getAsString(); + + switch (columnType) { + case BLOB: { + byte[] decodedBytes = Base64.getDecoder().decode(stringValue); + ByteArrayInputStream blob = new ByteArrayInputStream(decodedBytes); + stmt.setBlob(i, blob); + break; + } + case CLOB: + stmt.setClob(i, new StringReader(stringValue)); + break; + case INTEGER: + stmt.setInt(i, Integer.parseInt(stringValue)); + break; + case BIGINT: + stmt.setLong(i, Long.parseLong(stringValue)); + break; + case SMALLINT: + stmt.setShort(i, Short.parseShort(stringValue)); + break; + case DOUBLE: + stmt.setDouble(i, Double.parseDouble(stringValue)); + break; + case FLOAT: + stmt.setFloat(i, Float.parseFloat(stringValue)); + break; + case DECIMAL: + stmt.setBigDecimal(i, new BigDecimal(stringValue)); + break; + case DATE: + stmt.setDate(i, Date.valueOf(stringValue)); // expects ISO format: yyyy-[m]m-[d]d + break; + case TIME: + stmt.setTime(i, Time.valueOf(stringValue)); // expects format: hh:mm:ss + break; + case TIMESTAMP: + stmt.setTimestamp(i, Timestamp.valueOf(stringValue)); // expects format: yyyy-[m]m-[d]d hh:mm:ss[.f...] + break; + case BOOLEAN: + stmt.setBoolean(i, Boolean.parseBoolean(stringValue)); + break; + default: + stmt.setString(i, stringValue); // Fallback + } + } + + + public enum ColumnType { + CHAR, + VARCHAR, + INTEGER, + FLOAT, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + CLOB, + BLOB, + BOOLEAN, + SMALLINT, + BIGINT, + DOUBLE + } + + private ColumnType getColumnType(int i, JsonArray columnTypes){ + ColumnType columnType; + try { + columnType = ColumnType.valueOf(columnTypes.get(-1 + i).getAsString()); + } catch (IllegalArgumentException e){ + columnType = ColumnType.VARCHAR; + } + return columnType; + } + + private void addJsonArrayParameters(PreparedStatement stmt, JsonArray parameters, JsonArray columnTypes) throws SQLException { + for (int i = 1; i <= parameters.size(); ++i) { + JsonElement element = parameters.get(-1 + i); + ColumnType columnType = getColumnType(-1 + i, columnTypes); + if (element.isJsonNull()) { stmt.setNull(i, Types.NULL); } else { @@ -78,9 +166,9 @@ private void addJsonArrayParameters(PreparedStatement stmt, JsonArray arr) throw } else if (stmt instanceof CallableStatement && ParameterMetaData.parameterModeInOut == stmt.getParameterMetaData().getParameterMode(i)) { ((CallableStatement) stmt).registerOutParameter(i, stmt.getParameterMetaData().getParameterType(i)); - stmt.setString(i, element.getAsString()); + addParameter(i, element, columnType, stmt); } else { - stmt.setString(i, element.getAsString()); + addParameter(i, element, columnType, stmt); } } } From 8b6dc0486feba90df77cbd1e7971d80c80269454 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 13:07:48 -0400 Subject: [PATCH 03/57] fix bug in getcolumntype --- .../java/com/github/ibm/mapepire/requests/PreparedExecute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java index fa2d321..30ce70a 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java @@ -145,7 +145,7 @@ public enum ColumnType { private ColumnType getColumnType(int i, JsonArray columnTypes){ ColumnType columnType; try { - columnType = ColumnType.valueOf(columnTypes.get(-1 + i).getAsString()); + columnType = ColumnType.valueOf(columnTypes.get(i).getAsString()); } catch (IllegalArgumentException e){ columnType = ColumnType.VARCHAR; } From b5de3bbc7ca2325978c4775e2752e8719a5f77ca Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 17:10:03 -0400 Subject: [PATCH 04/57] send data diretly as raw bytes --- .../github/ibm/mapepire/requests/BlockRetrievableRequest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 911169c..9e7758b 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -1,5 +1,6 @@ package com.github.ibm.mapepire.requests; +import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -132,6 +133,9 @@ protected static DataBlockFetchResult getNextDataBlock(final ResultSet _rs, fina } } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; + } + else if (cellData instanceof Blob){ + cellDataForResponse = new String((byte[]) cellData, StandardCharsets.UTF_8); } else { cellDataForResponse = _rs.getString(col); } From 7a3ffef06260add71773e19c9580fc66e9d2bcbf Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 17:31:22 -0400 Subject: [PATCH 05/57] dont throw error if column types not exist --- .../ibm/mapepire/requests/BlockRetrievableRequest.java | 4 +--- .../github/ibm/mapepire/requests/PreparedExecute.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 9e7758b..c4328aa 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -134,9 +134,7 @@ protected static DataBlockFetchResult getNextDataBlock(final ResultSet _rs, fina } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; } - else if (cellData instanceof Blob){ - cellDataForResponse = new String((byte[]) cellData, StandardCharsets.UTF_8); - } else { + else { cellDataForResponse = _rs.getString(col); } if (_isTerseDataFormat) { diff --git a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java index 30ce70a..e9bb2b3 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java @@ -34,7 +34,13 @@ public PreparedExecute(final DataStreamProcessor _io, final JsonObject _reqObj, @Override protected void go() throws Exception { JsonArray parms = super.getRequestField("parameters").getAsJsonArray(); - JsonArray columnTypes = super.getRequestField("columnTypes").getAsJsonArray(); + JsonElement columnTypesElement = super.getRequestField("columnTypes"); + JsonArray columnTypes; + if (!columnTypesElement.isJsonNull()){ + columnTypes = columnTypesElement.getAsJsonArray(); + } else { + columnTypes = null; + } boolean isBatch = !parms.isEmpty() && parms.get(0).isJsonArray(); boolean hasResultSet = false; long batchUpdateCount = 0; @@ -155,7 +161,7 @@ private ColumnType getColumnType(int i, JsonArray columnTypes){ private void addJsonArrayParameters(PreparedStatement stmt, JsonArray parameters, JsonArray columnTypes) throws SQLException { for (int i = 1; i <= parameters.size(); ++i) { JsonElement element = parameters.get(-1 + i); - ColumnType columnType = getColumnType(-1 + i, columnTypes); + ColumnType columnType = columnTypes == null ? ColumnType.VARCHAR : getColumnType(-1 + i, columnTypes); if (element.isJsonNull()) { stmt.setNull(i, Types.NULL); From 31cc3f81d38916723f39117f4d2d03c12032c0e6 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 7 Aug 2025 17:45:37 -0400 Subject: [PATCH 06/57] fix null ptr --- .../java/com/github/ibm/mapepire/requests/PreparedExecute.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java index e9bb2b3..6057e20 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PreparedExecute.java @@ -36,7 +36,7 @@ protected void go() throws Exception { JsonArray parms = super.getRequestField("parameters").getAsJsonArray(); JsonElement columnTypesElement = super.getRequestField("columnTypes"); JsonArray columnTypes; - if (!columnTypesElement.isJsonNull()){ + if (columnTypesElement != null){ columnTypes = columnTypesElement.getAsJsonArray(); } else { columnTypes = null; From 3aa744575e8249554613e84bbc1b40289a77e0a4 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Fri, 8 Aug 2025 11:19:34 -0400 Subject: [PATCH 07/57] Increase max ws message size --- src/main/java/com/github/ibm/mapepire/MapepireServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/MapepireServer.java b/src/main/java/com/github/ibm/mapepire/MapepireServer.java index c3971a9..bdca704 100644 --- a/src/main/java/com/github/ibm/mapepire/MapepireServer.java +++ b/src/main/java/com/github/ibm/mapepire/MapepireServer.java @@ -170,7 +170,7 @@ public static void main(final String[] _args) { (servletContext, nativeWebSocketConfiguration) -> { nativeWebSocketConfiguration.getPolicy().setMaxTextMessageBufferSize(65535); // Configure max message size - int maxWsMessageSize = 50 * 1024 * 1024; // 50MB + int maxWsMessageSize = 200 * 1024 * 1024; // 50MB String maxWsMessageSizeStr = System.getenv("MAX_WS_MESSAGE_SIZE"); if (StringUtils.isNonEmpty(maxWsMessageSizeStr)) { maxWsMessageSize = Integer.parseInt(maxWsMessageSizeStr); From 4da583e077bb6a7bbe43ca4d38732faec237968a Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 11 Aug 2025 08:29:03 -0400 Subject: [PATCH 08/57] Try to put blob in prepared stmt --- .../ibm/mapepire/DataStreamProcessor.java | 36 +++++++++++++++++++ .../ibm/mapepire/ws/DbWebsocketClient.java | 10 ++++++ 2 files changed, 46 insertions(+) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index fe32592..a5c71ad 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -8,6 +8,7 @@ import java.io.*; import java.sql.SQLException; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -66,6 +67,41 @@ public void run() { } } + private int bytesToInt(byte[] bytes, int offset, int length) { + int x = 0; + for (int i = offset; i < offset + length; i++){ + byte b = bytes[i]; + x = x << 8 | b & 0xFF; + } + return x; + } + + + + public void run(byte[] binary) { + int cont_id = bytesToInt(binary, 0, 2); + int length = bytesToInt(binary, 2, 4); + + if (binary.length - 6 != length){ + throw new RuntimeException("Invalid binary data recieved."); + } + + + PrepareSql prev = m_prepStmtMap.get(cont_id); + if (null == prev) { + dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); + return; + } + byte[] blob = Arrays.copyOfRange(binary, 6, binary.length); + try { + RunBlob runBlob = new RunBlob(this, blob, prev); + + } catch (Exception e){ + + } + + } + public void run(String requestString) { final JsonElement reqElement; final JsonObject reqObj; diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 6dc0986..84f0ba7 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -8,6 +8,7 @@ import java.io.*; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; public class DbWebsocketClient extends WebSocketAdapter { @@ -33,6 +34,15 @@ public void onWebSocketText(String message) { io.run(message); } + @Override + public void onWebSocketBinary(byte[] payload, int offset, int len) { + // Access only the relevant portion of the data + byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); + io.run(binary); + + // Now use `message` as needed + } + @Override public void onWebSocketClose(int statusCode, String reason) { io.end(); From e98d9cc70d0536bf38c1c78fc372104733fbdd11 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 11 Aug 2025 09:07:45 -0400 Subject: [PATCH 09/57] add debug symbols --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index df1c68f..34a7991 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + true + lines,vars,source + + From 287be45decf3665d181c058c6fbc43564da57a24 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 11 Aug 2025 10:34:55 -0400 Subject: [PATCH 10/57] Add logging in onwebsocketbinary --- .../github/ibm/mapepire/requests/RunBlob.java | 92 +++++++++++++++++++ .../ibm/mapepire/ws/DbWebsocketClient.java | 1 + 2 files changed, 93 insertions(+) create mode 100644 src/main/java/com/github/ibm/mapepire/requests/RunBlob.java diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java new file mode 100644 index 0000000..302f7f9 --- /dev/null +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -0,0 +1,92 @@ +package com.github.ibm.mapepire.requests; + +import java.sql.*; +import java.util.Arrays; +import java.util.LinkedList; + +import com.github.ibm.mapepire.DataStreamProcessor; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import javax.sql.rowset.serial.SerialBlob; + +public class RunBlob{ + + private final PrepareSql m_prev; + + public RunBlob(final DataStreamProcessor _io, final byte[] binary, final PrepareSql _prev) throws SQLException { + m_prev = _prev; + PreparedStatement stmt = m_prev.getStatement(); + Blob blob = new SerialBlob(binary); +// stmt.setString(i, element.getAsString()); + stmt.setBlob(0, blob); + + stmt.getResultSet(); + + } + +// @Override +// protected void go() throws Exception { +// boolean isBatch = !parms.isEmpty() && parms.get(0).isJsonArray(); +// boolean hasResultSet = false; +// long batchUpdateCount = 0; +// +// PreparedStatement stmt = m_prev.getStatement(); +// if (isBatch) { +// JsonArray arr = parms.getAsJsonArray(); +// int batch_ops_added = 0; +// for (int i = 0; i < arr.size(); i++) { +// addJsonArrayParameters(stmt, arr.get(i).getAsJsonArray()); +// m_prev.getStatement().addBatch(); +// } +// batch_ops_added += arr.size(); +// addReplyData("batch_added", batch_ops_added); +// long updateCount[] = stmt.executeLargeBatch(); +// batchUpdateCount = Arrays.stream(updateCount).sum(); +// } else{ +// if (parms != null) { +// addJsonArrayParameters(stmt, parms.getAsJsonArray()); +// } +// hasResultSet = stmt.execute(); +// } +// +// if (hasResultSet) { +// this.m_rs = stmt.getResultSet(); +// final int numRows = super.getRequestFieldInt("rows", 1000); +// addReplyData("has_results", true); +// addReplyData("update_count", stmt.getLargeUpdateCount()); +// addReplyData("metadata", getResultMetaDataForResponse()); +// addReplyData("data", getNextDataBlock(numRows)); +// addReplyData("output_parms", getOutputParms(stmt)); +// addReplyData("is_done", isDone()); +// } else { +// addReplyData("data", new LinkedList()); +// addReplyData("has_results", false); +// addReplyData("update_count", batchUpdateCount != 0 ? batchUpdateCount : stmt.getLargeUpdateCount()); +// addReplyData("output_parms", getOutputParms(stmt)); +// addReplyData("is_done", m_isDone = true); +// } +// } + + private void addJsonArrayParameters(PreparedStatement stmt, JsonArray arr) throws SQLException { + for (int i = 1; i <= arr.size(); ++i) { + JsonElement element = arr.get(-1 + i); + if (element.isJsonNull()) { + stmt.setNull(i, Types.NULL); + } else { + if (stmt instanceof CallableStatement + && ParameterMetaData.parameterModeOut == stmt.getParameterMetaData().getParameterMode(i)) { + ((CallableStatement) stmt).registerOutParameter(i, stmt.getParameterMetaData().getParameterType(i)); + } else if (stmt instanceof CallableStatement + && ParameterMetaData.parameterModeInOut == stmt.getParameterMetaData().getParameterMode(i)) { + ((CallableStatement) stmt).registerOutParameter(i, stmt.getParameterMetaData().getParameterType(i)); + stmt.setString(i, element.getAsString()); + } else { + stmt.setString(i, element.getAsString()); + } + } + } + } + +} diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 84f0ba7..eea37c4 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -36,6 +36,7 @@ public void onWebSocketText(String message) { @Override public void onWebSocketBinary(byte[] payload, int offset, int len) { + System.out.println(">>> onWebSocketBinary called with len=" + len); // Access only the relevant portion of the data byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); io.run(binary); From eadeb0c1bee985f1d2dda79f6fa9a6ad772ab101 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 11 Aug 2025 11:38:47 -0400 Subject: [PATCH 11/57] Upgrade jetty version --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 34a7991..28b3a9a 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ UTF-8 yyyy-MM-dd HH:mm:ss - 9.4.54.v20240208 + 9.4.57.v20241219 install @@ -221,6 +221,7 @@ websocket-server ${jetty.version} + From 5e2ce8b28fbee9a24d013075109fa5f81a00e57a Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 12 Aug 2025 13:01:34 -0400 Subject: [PATCH 12/57] change maven compiler target --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 28b3a9a..d0bda6f 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,8 @@ UTF-8 yyyy-MM-dd HH:mm:ss 9.4.57.v20241219 + 1.8 + 1.8 install From 16e2f4cdf8d05e281dec8bab43fe63abd8806a0b Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 12 Aug 2025 13:12:36 -0400 Subject: [PATCH 13/57] change copy bytes --- src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index eea37c4..c655e83 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -38,7 +38,7 @@ public void onWebSocketText(String message) { public void onWebSocketBinary(byte[] payload, int offset, int len) { System.out.println(">>> onWebSocketBinary called with len=" + len); // Access only the relevant portion of the data - byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); + byte[] binary = Arrays.copyOfRange(payload, offset, offset + len - 1); io.run(binary); // Now use `message` as needed From a250d3f270159c6ff2ac96410551b859d6bf032f Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 12 Aug 2025 15:58:52 -0400 Subject: [PATCH 14/57] fix some bugs --- pom.xml | 4 ++++ .../java/com/github/ibm/mapepire/DataStreamProcessor.java | 5 ++--- .../java/com/github/ibm/mapepire/requests/RunBlob.java | 7 ++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d0bda6f..fac51f1 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,10 @@ org.apache.maven.plugins maven-compiler-plugin + + 8 + 8 + org.apache.maven.plugins diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index a5c71ad..bf1cc17 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -87,7 +87,7 @@ public void run(byte[] binary) { } - PrepareSql prev = m_prepStmtMap.get(cont_id); + PrepareSql prev = m_prepStmtMap.get(String.valueOf(cont_id)); if (null == prev) { dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); return; @@ -95,9 +95,8 @@ public void run(byte[] binary) { byte[] blob = Arrays.copyOfRange(binary, 6, binary.length); try { RunBlob runBlob = new RunBlob(this, blob, prev); - } catch (Exception e){ - + System.out.println("Caught exception " + e); } } diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 302f7f9..2acd40e 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -22,7 +22,12 @@ public RunBlob(final DataStreamProcessor _io, final byte[] binary, final Prepare // stmt.setString(i, element.getAsString()); stmt.setBlob(0, blob); - stmt.getResultSet(); + try { + stmt.getResultSet(); + + } catch (Exception e){ + System.out.println("Caught error " + e); + } } From d1cfb5e5cfaa9e31d471270cbadd7602cbd8f657 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 12 Aug 2025 16:49:54 -0400 Subject: [PATCH 15/57] fix some bugs --- src/main/java/com/github/ibm/mapepire/requests/RunBlob.java | 6 ++---- .../java/com/github/ibm/mapepire/ws/DbWebsocketClient.java | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 2acd40e..94e0cd7 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -19,11 +19,9 @@ public RunBlob(final DataStreamProcessor _io, final byte[] binary, final Prepare m_prev = _prev; PreparedStatement stmt = m_prev.getStatement(); Blob blob = new SerialBlob(binary); -// stmt.setString(i, element.getAsString()); - stmt.setBlob(0, blob); - + stmt.setBlob(1, blob); try { - stmt.getResultSet(); + int affectedRows = stmt.executeUpdate(); } catch (Exception e){ System.out.println("Caught error " + e); diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index c655e83..eea37c4 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -38,7 +38,7 @@ public void onWebSocketText(String message) { public void onWebSocketBinary(byte[] payload, int offset, int len) { System.out.println(">>> onWebSocketBinary called with len=" + len); // Access only the relevant portion of the data - byte[] binary = Arrays.copyOfRange(payload, offset, offset + len - 1); + byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); io.run(binary); // Now use `message` as needed From 24718eea23e2bfef7cf6d16522f5172b1da4b18d Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 09:48:46 -0400 Subject: [PATCH 16/57] Increase max binary message size --- .../java/com/github/ibm/mapepire/MapepireServer.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/MapepireServer.java b/src/main/java/com/github/ibm/mapepire/MapepireServer.java index bdca704..62201e2 100644 --- a/src/main/java/com/github/ibm/mapepire/MapepireServer.java +++ b/src/main/java/com/github/ibm/mapepire/MapepireServer.java @@ -39,6 +39,10 @@ public static boolean isSingleMode() { return s_isSingleMode; } + private static int getNumBytesInMb(int num){ + return num * 1024 * 1024; + } + public static void main(final String[] _args) { final LinkedList args = new LinkedList(); @@ -170,12 +174,15 @@ public static void main(final String[] _args) { (servletContext, nativeWebSocketConfiguration) -> { nativeWebSocketConfiguration.getPolicy().setMaxTextMessageBufferSize(65535); // Configure max message size - int maxWsMessageSize = 200 * 1024 * 1024; // 50MB + int maxWsMessageSize = getNumBytesInMb(200); + int maxBinaryMessageSize = getNumBytesInMb(200); String maxWsMessageSizeStr = System.getenv("MAX_WS_MESSAGE_SIZE"); if (StringUtils.isNonEmpty(maxWsMessageSizeStr)) { maxWsMessageSize = Integer.parseInt(maxWsMessageSizeStr); } nativeWebSocketConfiguration.getPolicy().setMaxTextMessageSize(maxWsMessageSize); + nativeWebSocketConfiguration.getPolicy().setMaxBinaryMessageSize(maxBinaryMessageSize); + // Add websockets nativeWebSocketConfiguration.addMapping("/db/*", new DbSocketCreator()); From 55f9e58f84e84d9beab3bd2fcb2d57c0eee38c4c Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 09:58:31 -0400 Subject: [PATCH 17/57] Use byte array input stream instead --- .../com/github/ibm/mapepire/requests/RunBlob.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 94e0cd7..54bdecd 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -1,5 +1,7 @@ package com.github.ibm.mapepire.requests; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.sql.*; import java.util.Arrays; import java.util.LinkedList; @@ -18,12 +20,11 @@ public class RunBlob{ public RunBlob(final DataStreamProcessor _io, final byte[] binary, final PrepareSql _prev) throws SQLException { m_prev = _prev; PreparedStatement stmt = m_prev.getStatement(); - Blob blob = new SerialBlob(binary); - stmt.setBlob(1, blob); - try { + try (InputStream is = new ByteArrayInputStream(binary)) { + stmt.setBinaryStream(1, is, binary.length); int affectedRows = stmt.executeUpdate(); - - } catch (Exception e){ + } + catch (Exception e){ System.out.println("Caught error " + e); } From 8990f2bd956741deaff3dd760b5e0f3866378af5 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 10:19:52 -0400 Subject: [PATCH 18/57] dont copy payload around --- .../ibm/mapepire/DataStreamProcessor.java | 49 ++++++++++++++----- .../github/ibm/mapepire/requests/RunBlob.java | 6 +-- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index bf1cc17..68204f7 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -24,7 +24,7 @@ public class DataStreamProcessor implements Runnable { private final boolean m_isTestMode; public DataStreamProcessor(final InputStream _in, final PrintStream _out, final SystemConnection _conn, - boolean _isTestMode) + boolean _isTestMode) throws UnsupportedEncodingException { m_in = new BufferedReader(new InputStreamReader(_in, "UTF-8")); m_out = _out; @@ -69,7 +69,7 @@ public void run() { private int bytesToInt(byte[] bytes, int offset, int length) { int x = 0; - for (int i = offset; i < offset + length; i++){ + for (int i = offset; i < offset + length; i++) { byte b = bytes[i]; x = x << 8 | b & 0xFF; } @@ -77,12 +77,11 @@ private int bytesToInt(byte[] bytes, int offset, int length) { } + public void run(byte[] payload, int offset, int len) { + int cont_id = bytesToInt(payload, offset, 2); + int length = bytesToInt(payload, offset + 2, 4); - public void run(byte[] binary) { - int cont_id = bytesToInt(binary, 0, 2); - int length = bytesToInt(binary, 2, 4); - - if (binary.length - 6 != length){ + if (payload.length - 6 != length) { throw new RuntimeException("Invalid binary data recieved."); } @@ -92,14 +91,38 @@ public void run(byte[] binary) { dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); return; } - byte[] blob = Arrays.copyOfRange(binary, 6, binary.length); - try { - RunBlob runBlob = new RunBlob(this, blob, prev); - } catch (Exception e){ - System.out.println("Caught exception " + e); +// byte[] blob = Arrays.copyOfRange(payload, 6, payload.length); + int blobOffset = 6; + try { + RunBlob runBlob = new RunBlob(this, payload, offset, len, prev); + } catch (Exception e) { + System.out.println("Caught exception " + e); + } } - } + +// public void run(byte[] binary) { +// int cont_id = bytesToInt(binary, 0, 2); +// int length = bytesToInt(binary, 2, 4); +// +// if (binary.length - 6 != length) { +// throw new RuntimeException("Invalid binary data recieved."); +// } +// +// +// PrepareSql prev = m_prepStmtMap.get(String.valueOf(cont_id)); +// if (null == prev) { +// dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); +// return; +// } +// byte[] blob = Arrays.copyOfRange(binary, 6, binary.length); +// try { +// RunBlob runBlob = new RunBlob(this, blob, prev); +// } catch (Exception e) { +// System.out.println("Caught exception " + e); +// } +// +// } public void run(String requestString) { final JsonElement reqElement; diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 54bdecd..1e2bc9d 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -17,11 +17,11 @@ public class RunBlob{ private final PrepareSql m_prev; - public RunBlob(final DataStreamProcessor _io, final byte[] binary, final PrepareSql _prev) throws SQLException { + public RunBlob(final DataStreamProcessor _io, final byte[] binary, final int offset, final int length, final PrepareSql _prev) throws SQLException { m_prev = _prev; PreparedStatement stmt = m_prev.getStatement(); - try (InputStream is = new ByteArrayInputStream(binary)) { - stmt.setBinaryStream(1, is, binary.length); + try (InputStream is = new ByteArrayInputStream(binary, offset, length)) { + stmt.setBinaryStream(1, is, length); int affectedRows = stmt.executeUpdate(); } catch (Exception e){ From 509d982716ac7a407c9800d2c831a1e7aed34010 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 10:21:07 -0400 Subject: [PATCH 19/57] fix bug --- .../java/com/github/ibm/mapepire/ws/DbWebsocketClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index eea37c4..4e574a5 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -38,8 +38,8 @@ public void onWebSocketText(String message) { public void onWebSocketBinary(byte[] payload, int offset, int len) { System.out.println(">>> onWebSocketBinary called with len=" + len); // Access only the relevant portion of the data - byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); - io.run(binary); +// byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); + io.run(payload, offset, len); // Now use `message` as needed } From 78ebb85fa8fd41dfe079ffe3b293830185f1cefb Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 10:31:45 -0400 Subject: [PATCH 20/57] use bloboffset --- src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 68204f7..00df5e6 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -94,7 +94,7 @@ public void run(byte[] payload, int offset, int len) { // byte[] blob = Arrays.copyOfRange(payload, 6, payload.length); int blobOffset = 6; try { - RunBlob runBlob = new RunBlob(this, payload, offset, len, prev); + RunBlob runBlob = new RunBlob(this, payload, blobOffset, len, prev); } catch (Exception e) { System.out.println("Caught exception " + e); } From 4861c2e2cf201bff2780d299b09a23316f07df37 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 10:36:13 -0400 Subject: [PATCH 21/57] use proper length --- src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 00df5e6..4910445 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -94,7 +94,7 @@ public void run(byte[] payload, int offset, int len) { // byte[] blob = Arrays.copyOfRange(payload, 6, payload.length); int blobOffset = 6; try { - RunBlob runBlob = new RunBlob(this, payload, blobOffset, len, prev); + RunBlob runBlob = new RunBlob(this, payload, blobOffset, length, prev); } catch (Exception e) { System.out.println("Caught exception " + e); } From e837264162c5c136c3134c67323082a18bed1084 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 13 Aug 2025 11:21:39 -0400 Subject: [PATCH 22/57] dont convert blob to string --- .../github/ibm/mapepire/requests/BlockRetrievableRequest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index c4328aa..7907288 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -1,5 +1,6 @@ package com.github.ibm.mapepire.requests; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.LinkedHashMap; @@ -11,6 +12,7 @@ import com.github.ibm.mapepire.DataStreamProcessor; import com.github.ibm.mapepire.SystemConnection; import com.google.gson.JsonObject; +import com.ibm.as400.access.AS400JDBCBlobLocator; import com.ibm.as400.access.AS400JDBCParameterMetaData; public abstract class BlockRetrievableRequest extends ClientRequest { @@ -133,6 +135,8 @@ protected static DataBlockFetchResult getNextDataBlock(final ResultSet _rs, fina } } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; + } else if (cellData instanceof AS400JDBCBlobLocator){ + cellDataForResponse = _rs.getBytes(col); } else { cellDataForResponse = _rs.getString(col); From 36ffc1281929dfd392557b9823829f74d1f45e0b Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 09:29:57 -0400 Subject: [PATCH 23/57] Send response back as blob --- .../github/ibm/mapepire/ClientRequest.java | 6 +--- .../ibm/mapepire/DataStreamProcessor.java | 32 ++++++++++++++++--- .../java/com/github/ibm/mapepire/Version.java | 4 +-- .../requests/BlockRetrievableRequest.java | 14 +++++--- .../github/ibm/mapepire/requests/Exit.java | 2 +- .../ibm/mapepire/ws/DbWebsocketClient.java | 20 ++++++++---- 6 files changed, 55 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/ClientRequest.java b/src/main/java/com/github/ibm/mapepire/ClientRequest.java index 31399cd..a25d9fb 100644 --- a/src/main/java/com/github/ibm/mapepire/ClientRequest.java +++ b/src/main/java/com/github/ibm/mapepire/ClientRequest.java @@ -16,7 +16,7 @@ public abstract class ClientRequest implements Runnable { private final SystemConnection m_conn; private final String m_id; - private final DataStreamProcessor m_io; + protected final DataStreamProcessor m_io; private final JsonObject m_reqObj; private final Map replyData = new LinkedHashMap(); @@ -28,10 +28,6 @@ protected ClientRequest(final DataStreamProcessor _io, final SystemConnection _c addReplyData("id", m_id); } - public SystemConnection getConnection() { - return m_conn; - } - protected void addReplyData(final String _key, final Object _val) { replyData.put(_key, _val); } diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 4910445..fc4f8f6 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -1,12 +1,14 @@ package com.github.ibm.mapepire; import com.github.ibm.mapepire.requests.*; +import com.github.ibm.mapepire.ws.DbWebsocketClient; import com.github.theprez.jcmdutils.StringUtils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.io.*; +import java.nio.ByteBuffer; import java.sql.SQLException; import java.util.Arrays; import java.util.HashMap; @@ -18,16 +20,19 @@ public class DataStreamProcessor implements Runnable { private final SystemConnection m_conn; private final BufferedReader m_in; - private final PrintStream m_out; + private final PrintStream m_out_text; + private final Map m_queriesMap = new HashMap(); private final Map m_prepStmtMap = new HashMap(); private final boolean m_isTestMode; + private final DbWebsocketClient.BinarySender m_binarySender; - public DataStreamProcessor(final InputStream _in, final PrintStream _out, final SystemConnection _conn, + public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final DbWebsocketClient.BinarySender binarySender, final SystemConnection _conn, boolean _isTestMode) throws UnsupportedEncodingException { m_in = new BufferedReader(new InputStreamReader(_in, "UTF-8")); - m_out = _out; + m_out_text = _outText; + m_binarySender = binarySender; m_conn = _conn; m_isTestMode = _isTestMode; } @@ -243,9 +248,26 @@ public void run(String requestString) { public void sendResponse(final String _response) throws UnsupportedEncodingException, IOException { synchronized (s_replyWriterLock) { - m_out.write((_response + "\n").getBytes("UTF-8")); + m_out_text.write((_response + "\n").getBytes("UTF-8")); Tracer.datastreamOut(_response); - m_out.flush(); + m_out_text.flush(); + } + } + + public void sendResponse(final InputStream is) throws UnsupportedEncodingException, IOException { + synchronized (s_replyWriterLock) { + byte[] buffer = new byte[8192]; + int bytesRead; + boolean isFinal; + while ((bytesRead = is.read(buffer)) != -1) { + // Wrap only the bytes actually read + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); + + // Check if this is the last chunk + isFinal = is.available() == 0; + + m_binarySender.send(byteBuffer, isFinal); + } } } diff --git a/src/main/java/com/github/ibm/mapepire/Version.java b/src/main/java/com/github/ibm/mapepire/Version.java index 225954b..91902c0 100644 --- a/src/main/java/com/github/ibm/mapepire/Version.java +++ b/src/main/java/com/github/ibm/mapepire/Version.java @@ -1,5 +1,5 @@ package com.github.ibm.mapepire; public class Version { - static public final String s_compileDateTime = "2024-08-08 00:36:20 (GMT)"; - static public final String s_version = "2.0.0-rc1"; + static public final String s_compileDateTime = "2025-08-12 20:50:37 (GMT)"; + static public final String s_version = "2.3.3"; } \ No newline at end of file diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 7907288..a6179f0 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -1,5 +1,6 @@ package com.github.ibm.mapepire.requests; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.*; @@ -20,13 +21,15 @@ public abstract class BlockRetrievableRequest extends ClientRequest { protected boolean m_isDone = false; protected ResultSet m_rs = null; protected final boolean m_isTerseData; + private DataStreamProcessor m_io; protected BlockRetrievableRequest(DataStreamProcessor _io, SystemConnection _conn, JsonObject _reqObj) { super(_io, _conn, _reqObj); + m_io = _io; m_isTerseData = getRequestFieldBoolean("terse", false); } - List getNextDataBlock(final int _numRows) throws SQLException { + List getNextDataBlock(final int _numRows) throws SQLException, IOException { if (m_isDone) { return new LinkedList(); } @@ -97,8 +100,8 @@ public Object getData() { } } - protected static DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _numRows, - final boolean _isTerseDataFormat) throws SQLException { + protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _numRows, + final boolean _isTerseDataFormat) throws SQLException, IOException { final DataBlockFetchResult ret = new DataBlockFetchResult(); if (null == _rs) { @@ -136,7 +139,10 @@ protected static DataBlockFetchResult getNextDataBlock(final ResultSet _rs, fina } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; } else if (cellData instanceof AS400JDBCBlobLocator){ - cellDataForResponse = _rs.getBytes(col); + InputStream is = _rs.getBinaryStream(col); + m_io.sendResponse(is); + +// cellDataForResponse = _rs.getBytes(col); } else { cellDataForResponse = _rs.getString(col); diff --git a/src/main/java/com/github/ibm/mapepire/requests/Exit.java b/src/main/java/com/github/ibm/mapepire/requests/Exit.java index eb77edb..74fe75e 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/Exit.java +++ b/src/main/java/com/github/ibm/mapepire/requests/Exit.java @@ -21,7 +21,7 @@ protected void go() throws Exception { @Override protected void processAfterReplySent() { if (DbSocketCreator.isDaemon()) { - this.getConnection().close(); + this.getSystemConnection().close(); } else { Tracer.info("exiting as requested"); System.exit(0); diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 4e574a5..1450f3d 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -2,23 +2,30 @@ import com.github.ibm.mapepire.DataStreamProcessor; import com.github.ibm.mapepire.SystemConnection; +import org.eclipse.jetty.websocket.api.RemoteEndpoint; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.api.WebSocketAdapter; import org.eclipse.jetty.websocket.api.WebSocketException; import java.io.*; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.concurrent.CountDownLatch; public class DbWebsocketClient extends WebSocketAdapter { private final CountDownLatch closureLatch = new CountDownLatch(1); private final DataStreamProcessor io; + private final RemoteEndpoint remote; + + @FunctionalInterface + public interface BinarySender { + void send(ByteBuffer buffer, boolean isLast) throws IOException; + } DbWebsocketClient(String clientHost, String clientAddress, String host, String user, String pass) throws IOException { super(); + remote = getRemote(); SystemConnection conn = new SystemConnection(clientHost, clientAddress,host, user, pass); - io = getDataStream(this, conn); + io = getDataStreamProcessor(this, conn); } @Override @@ -62,10 +69,11 @@ public void awaitClosure() throws InterruptedException { closureLatch.await(); } - private static DataStreamProcessor getDataStream(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { + private DataStreamProcessor getDataStreamProcessor(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { + BinarySender binarySender = (data, isLast) -> remote.sendPartialBytes(data, isLast); InputStream in = new ByteArrayInputStream(new byte[0]); - OutputStream outStream = new OutputStream() { + OutputStream outStreamText = new OutputStream() { private final ByteArrayOutputStream payload = new ByteArrayOutputStream(); @Override @@ -93,8 +101,8 @@ public synchronized void flush() throws IOException { } }; - PrintStream out = new PrintStream(outStream); + PrintStream out = new PrintStream(outStreamText); - return new DataStreamProcessor(in, out, conn, false); + return new DataStreamProcessor(in, out, binarySender, conn, false); } } From 93a9b92343f61be3f13b7225d45d404132345ba2 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 09:42:32 -0400 Subject: [PATCH 24/57] comment out some errors with binary sender --- src/main/java/com/github/ibm/mapepire/MapepireServer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/MapepireServer.java b/src/main/java/com/github/ibm/mapepire/MapepireServer.java index 62201e2..7515f99 100644 --- a/src/main/java/com/github/ibm/mapepire/MapepireServer.java +++ b/src/main/java/com/github/ibm/mapepire/MapepireServer.java @@ -10,6 +10,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import com.github.ibm.mapepire.ws.DbWebsocketClient; import org.eclipse.jetty.security.ConstraintMapping; import org.eclipse.jetty.security.ConstraintSecurityHandler; import org.eclipse.jetty.server.Server; @@ -67,9 +68,10 @@ public static void main(final String[] _args) { if (testMode) { System.setIn(new FileInputStream(testFile)); } - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, conn, testMode); +// DbWebsocketClient.BinarySender binarySender = (data, isLast) -> remote.sendPartialBytes(data, isLast); +// final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, conn, testMode); - io.run(); +// io.run(); } else { s_isSingleMode = false; From 6ee37d13c8472f1ab0979bae4bd649db18a4fca3 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 10:08:20 -0400 Subject: [PATCH 25/57] Check if instance of blob --- .../github/ibm/mapepire/requests/BlockRetrievableRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index a6179f0..b845987 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -138,7 +138,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; - } else if (cellData instanceof AS400JDBCBlobLocator){ + } else if (cellData instanceof Blob){ InputStream is = _rs.getBinaryStream(col); m_io.sendResponse(is); From 313ba36500d4ace0a5d29bfc92b8541aef78f797 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 10:33:01 -0400 Subject: [PATCH 26/57] get remote in method --- .../java/com/github/ibm/mapepire/ws/DbWebsocketClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 1450f3d..3c7197d 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -14,7 +14,6 @@ public class DbWebsocketClient extends WebSocketAdapter { private final CountDownLatch closureLatch = new CountDownLatch(1); private final DataStreamProcessor io; - private final RemoteEndpoint remote; @FunctionalInterface public interface BinarySender { @@ -70,7 +69,7 @@ public void awaitClosure() throws InterruptedException { } private DataStreamProcessor getDataStreamProcessor(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { - BinarySender binarySender = (data, isLast) -> remote.sendPartialBytes(data, isLast); + BinarySender binarySender = (data, isLast) -> getRemote().sendPartialBytes(data, isLast); InputStream in = new ByteArrayInputStream(new byte[0]); OutputStream outStreamText = new OutputStream() { From 6e70c900a3897d96ebd213506713a3017fd0179e Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 10:33:52 -0400 Subject: [PATCH 27/57] fix bug --- src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 3c7197d..d4cc1d1 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -22,7 +22,6 @@ public interface BinarySender { DbWebsocketClient(String clientHost, String clientAddress, String host, String user, String pass) throws IOException { super(); - remote = getRemote(); SystemConnection conn = new SystemConnection(clientHost, clientAddress,host, user, pass); io = getDataStreamProcessor(this, conn); } From 22cc52b25a355fc81933adc00b9a3f8ea6d74e21 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 12:37:02 -0400 Subject: [PATCH 28/57] send data id back --- .../ibm/mapepire/DataStreamProcessor.java | 47 +++++++++++++++---- .../requests/BlockRetrievableRequest.java | 3 +- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index fc4f8f6..cf8d1da 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -254,23 +254,52 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep } } - public void sendResponse(final InputStream is) throws UnsupportedEncodingException, IOException { +// public void sendResponse(final InputStream is, final String id) throws UnsupportedEncodingException, IOException { +// synchronized (s_replyWriterLock) { +// byte[] buffer = new byte[8192]; +// int bytesRead; +// boolean isFinal; +// while ((bytesRead = is.read(buffer)) != -1) { +// // Wrap only the bytes actually read +// ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); +// +// // Check if this is the last chunk +// isFinal = is.available() == 0; +// +// m_binarySender.send(byteBuffer, isFinal); +// } +// } +// } + + public void sendResponse(final InputStream is, final String id) throws UnsupportedEncodingException, IOException { synchronized (s_replyWriterLock) { byte[] buffer = new byte[8192]; - int bytesRead; - boolean isFinal; - while ((bytesRead = is.read(buffer)) != -1) { - // Wrap only the bytes actually read - ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); + short idValue = Short.parseShort(id); + byte[] idBytes = ByteBuffer.allocate(2).putShort(idValue).array(); + System.arraycopy(idBytes, 0, buffer, 0, idBytes.length); - // Check if this is the last chunk - isFinal = is.available() == 0; + int bytesRead; + bytesRead = is.read(buffer, idBytes.length, buffer.length - idBytes.length); + if (bytesRead != -1){ + sendByteBuffer(buffer, bytesRead + idBytes.length, is); + } - m_binarySender.send(byteBuffer, isFinal); + while ((bytesRead = is.read(buffer)) != -1) { + sendByteBuffer(buffer, bytesRead, is); } } } + private void sendByteBuffer(byte[] buffer, int bytesRead, InputStream is) throws IOException { + // Wrap only the bytes actually read + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); + + // Check if this is the last chunk + boolean isFinal = is.available() == 0; + + m_binarySender.send(byteBuffer, isFinal); + } + public void end() { try { m_conn.getJdbcConnection().close(); diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index b845987..5b2e6a9 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -139,8 +139,9 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; } else if (cellData instanceof Blob){ + String id = this.getId(); InputStream is = _rs.getBinaryStream(col); - m_io.sendResponse(is); + m_io.sendResponse(is, id); // cellDataForResponse = _rs.getBytes(col); } From 6fbaec6286ae39df35614472de9e1a33a3be445e Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 12:50:37 -0400 Subject: [PATCH 29/57] id is actually a string --- .../java/com/github/ibm/mapepire/DataStreamProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index cf8d1da..3b2a7b4 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -9,6 +9,7 @@ import java.io.*; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.Arrays; import java.util.HashMap; @@ -274,8 +275,7 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep public void sendResponse(final InputStream is, final String id) throws UnsupportedEncodingException, IOException { synchronized (s_replyWriterLock) { byte[] buffer = new byte[8192]; - short idValue = Short.parseShort(id); - byte[] idBytes = ByteBuffer.allocate(2).putShort(idValue).array(); + byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); System.arraycopy(idBytes, 0, buffer, 0, idBytes.length); int bytesRead; From 062f05c93304403355a015f18629da24e78fbac7 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 14 Aug 2025 13:02:27 -0400 Subject: [PATCH 30/57] add id length as first byte --- .../ibm/mapepire/DataStreamProcessor.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 3b2a7b4..4b0241b 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -272,16 +272,23 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final InputStream is, final String id) throws UnsupportedEncodingException, IOException { + public void sendResponse(final InputStream is, final String id) throws IOException { synchronized (s_replyWriterLock) { byte[] buffer = new byte[8192]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); - System.arraycopy(idBytes, 0, buffer, 0, idBytes.length); - int bytesRead; - bytesRead = is.read(buffer, idBytes.length, buffer.length - idBytes.length); - if (bytesRead != -1){ - sendByteBuffer(buffer, bytesRead + idBytes.length, is); + if (idBytes.length > 255) { + throw new IllegalArgumentException("ID too long to encode in one byte length"); + } + + // First byte is the length of the ID + buffer[0] = (byte) idBytes.length; + // Copy ID bytes after the length byte + System.arraycopy(idBytes, 0, buffer, 1, idBytes.length); + + int bytesRead = is.read(buffer, 1 + idBytes.length, buffer.length - (1 + idBytes.length)); + if (bytesRead != -1) { + sendByteBuffer(buffer, bytesRead + 1 + idBytes.length, is); } while ((bytesRead = is.read(buffer)) != -1) { From e1dd33f68cb4095eacb4ca8c3deb29826ea2c82b Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Sun, 17 Aug 2025 16:52:11 -0400 Subject: [PATCH 31/57] Got blob test kinda working --- .../ibm/mapepire/DataStreamProcessor.java | 28 +- .../requests/BlockRetrievableRequest.java | 2 +- .../github/ibm/mapepire/ws/BinarySender.java | 9 + .../ibm/mapepire/ws/DbWebsocketClient.java | 5 +- src/test/java/BlobTest.java | 164 +++ .../java/BlockRetrievableRequestImpl.java | 22 + src/test/java/MockResultSet.java | 1032 +++++++++++++++++ 7 files changed, 1253 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/github/ibm/mapepire/ws/BinarySender.java create mode 100644 src/test/java/BlobTest.java create mode 100644 src/test/java/BlockRetrievableRequestImpl.java create mode 100644 src/test/java/MockResultSet.java diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 4b0241b..8597b4b 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -1,6 +1,7 @@ package com.github.ibm.mapepire; import com.github.ibm.mapepire.requests.*; +import com.github.ibm.mapepire.ws.BinarySender; import com.github.ibm.mapepire.ws.DbWebsocketClient; import com.github.theprez.jcmdutils.StringUtils; import com.google.gson.JsonElement; @@ -26,9 +27,10 @@ public class DataStreamProcessor implements Runnable { private final Map m_queriesMap = new HashMap(); private final Map m_prepStmtMap = new HashMap(); private final boolean m_isTestMode; - private final DbWebsocketClient.BinarySender m_binarySender; + private final BinarySender m_binarySender; +// private final DbWebsocketClient.BinarySender m_binarySender; - public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final DbWebsocketClient.BinarySender binarySender, final SystemConnection _conn, + public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final BinarySender binarySender, final SystemConnection _conn, boolean _isTestMode) throws UnsupportedEncodingException { m_in = new BufferedReader(new InputStreamReader(_in, "UTF-8")); @@ -272,10 +274,12 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final InputStream is, final String id) throws IOException { + public void sendResponse(final InputStream is, final String id, final String column) throws IOException { synchronized (s_replyWriterLock) { + int curOffset = 0; byte[] buffer = new byte[8192]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); + byte[] columnNameBytes = column.getBytes(StandardCharsets.UTF_8); if (idBytes.length > 255) { throw new IllegalArgumentException("ID too long to encode in one byte length"); @@ -283,12 +287,24 @@ public void sendResponse(final InputStream is, final String id) throws IOExcepti // First byte is the length of the ID buffer[0] = (byte) idBytes.length; + curOffset += 1; // Copy ID bytes after the length byte - System.arraycopy(idBytes, 0, buffer, 1, idBytes.length); + System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); + curOffset += idBytes.length; - int bytesRead = is.read(buffer, 1 + idBytes.length, buffer.length - (1 + idBytes.length)); + // Copy column length + buffer[curOffset] = (byte) column.length(); + curOffset += 1; + + // copy column name + System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); + curOffset += columnNameBytes.length; + + + int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); + curOffset += bytesRead; if (bytesRead != -1) { - sendByteBuffer(buffer, bytesRead + 1 + idBytes.length, is); + sendByteBuffer(buffer, curOffset, is); } while ((bytesRead = is.read(buffer)) != -1) { diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 5b2e6a9..6985373 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -141,7 +141,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Blob){ String id = this.getId(); InputStream is = _rs.getBinaryStream(col); - m_io.sendResponse(is, id); + m_io.sendResponse(is, id, column); // cellDataForResponse = _rs.getBytes(col); } diff --git a/src/main/java/com/github/ibm/mapepire/ws/BinarySender.java b/src/main/java/com/github/ibm/mapepire/ws/BinarySender.java new file mode 100644 index 0000000..503722a --- /dev/null +++ b/src/main/java/com/github/ibm/mapepire/ws/BinarySender.java @@ -0,0 +1,9 @@ +package com.github.ibm.mapepire.ws; + +import java.io.IOException; +import java.nio.ByteBuffer; + +@FunctionalInterface +public interface BinarySender { + void send(ByteBuffer buffer, boolean isLast) throws IOException; +} \ No newline at end of file diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index d4cc1d1..1473c68 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -10,15 +10,16 @@ import java.io.*; import java.nio.ByteBuffer; import java.util.concurrent.CountDownLatch; +import com.github.ibm.mapepire.ws.BinarySender; public class DbWebsocketClient extends WebSocketAdapter { private final CountDownLatch closureLatch = new CountDownLatch(1); private final DataStreamProcessor io; - @FunctionalInterface + /* @FunctionalInterface public interface BinarySender { void send(ByteBuffer buffer, boolean isLast) throws IOException; - } + }*/ DbWebsocketClient(String clientHost, String clientAddress, String host, String user, String pass) throws IOException { super(); diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java new file mode 100644 index 0000000..2b2a022 --- /dev/null +++ b/src/test/java/BlobTest.java @@ -0,0 +1,164 @@ +import com.github.ibm.mapepire.DataStreamProcessor; +import com.github.ibm.mapepire.requests.BlockRetrievableRequest; +import com.github.ibm.mapepire.ws.BinarySender; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.*; +import javax.sql.rowset.serial.SerialBlob; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.sql.Blob; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +public class BlobTest { + + @Test + public void testMockResultSetSingleRow() throws SQLException { + // Arrange + Map row = new HashMap<>(); + List> rowsByIndex; + + row.put("id", 1); + row.put("name", "Alice"); + + List> rows = Collections.singletonList(row); + ResultSet rs = new MockResultSet(rows, null); + + // Act & Assert + assertTrue(rs.next(), "Expected at least one row"); + assertEquals(1, rs.getInt("id")); + assertEquals("Alice", rs.getString("name")); + + assertFalse(rs.next(), "No more rows expected"); + } + + @Test + public void testMockResultSetMultipleRows() throws SQLException { + // Arrange + Map row1 = new HashMap<>(); + row1.put("id", 1); + row1.put("name", "Alice"); + + Map row2 = new HashMap<>(); + row2.put("id", 2); + row2.put("name", "Bob"); + + List> rows = Arrays.asList(row1, row2); + ResultSet rs = new MockResultSet(rows, null); + + // First row + assertTrue(rs.next()); + assertEquals(1, rs.getInt("id")); + assertEquals("Alice", rs.getString("name")); + + // Second row + assertTrue(rs.next()); + assertEquals(2, rs.getInt("id")); + assertEquals("Bob", rs.getString("name")); + + // No more + assertFalse(rs.next()); + } + + @Test + void testBlobColumnInResultSet() throws Exception { + // Create some fake binary data + byte[] data = "hello world".getBytes(); + + // Wrap it in a Blob + Blob blob = new SerialBlob(data); + + List> rowsByIndex = Arrays.asList(Arrays.asList(blob)); + + + // Use MockResultSet to simulate a DB result + MockResultSet rs = new MockResultSet(rowsByIndex); + + // Act: move to the first row + assertTrue(rs.next()); + + // Retrieve the blob + Blob retrievedBlob = rs.getBlob(1); + assertNotNull(retrievedBlob); + + // Read the bytes back + byte[] retrievedData = retrievedBlob.getBytes(1, (int) retrievedBlob.length()); + + // Verify the round-trip matches + assertArrayEquals(data, retrievedData); + + // No more rows + assertFalse(rs.next()); + } + + @Test + public void testGetBinaryStreamByIndex() throws SQLException, IOException { + // Prepare blob data + byte[] blobData = {1, 2, 3, 4, 5}; + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList("Alice", new ByteArrayInputStream(blobData))); + + MockResultSet rs = new MockResultSet(rowsByIndex); + + assertTrue(rs.next(), "Should have first row"); + + InputStream is = rs.getBinaryStream(2); // second column + assertNotNull(is, "BinaryStream should not be null"); + + byte[] readData = is.readAllBytes(); + assertArrayEquals(blobData, readData, "Binary data should match"); + } + + @Test + public void testBlockRetrievableReq() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "12345"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blobData = {1, 2, 3, 4, 5}; + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 4, 98, 108, 111, 98, 1, 2, 3, 4, 5}; + Blob blob = new SerialBlob(blobData); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob)); + + Map row1 = new HashMap<>(); + row1.put("blob", blob); + List> rows = Arrays.asList(row1); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 1, false); + + verify(binarySender, times(1)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); + + // Verify first call + ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); + Boolean firstFlag = booleanCaptor.getAllValues().get(0); + byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); + assertArrayEquals(expectedResult, actual); + assertTrue(firstFlag); + + } + + +} diff --git a/src/test/java/BlockRetrievableRequestImpl.java b/src/test/java/BlockRetrievableRequestImpl.java new file mode 100644 index 0000000..6eafa68 --- /dev/null +++ b/src/test/java/BlockRetrievableRequestImpl.java @@ -0,0 +1,22 @@ +import com.github.ibm.mapepire.DataStreamProcessor; +import com.github.ibm.mapepire.SystemConnection; +import com.github.ibm.mapepire.requests.BlockRetrievableRequest; +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class BlockRetrievableRequestImpl extends BlockRetrievableRequest { + protected BlockRetrievableRequestImpl(DataStreamProcessor _io, SystemConnection _conn, JsonObject _reqObj) { + super(_io, _conn, _reqObj); + } + + public DataBlockFetchResult getNextDataBlockUsage(final ResultSet _rs, final int _numRows, + final boolean _isTerseDataFormat) throws SQLException, IOException { + return getNextDataBlock(_rs, _numRows, _isTerseDataFormat); + } + @Override + protected void go() throws Exception { + } +} diff --git a/src/test/java/MockResultSet.java b/src/test/java/MockResultSet.java new file mode 100644 index 0000000..d5f4d57 --- /dev/null +++ b/src/test/java/MockResultSet.java @@ -0,0 +1,1032 @@ +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.sql.Date; +import java.util.*; + +public class MockResultSet implements ResultSet { + private List> rows = null; + private List> rowsByIndex = null; + + private int cursor = -1; + + public MockResultSet(List> rows, List> rowsByIndex) { + this.rows = rows; + this.rowsByIndex = rowsByIndex; + } + + + public MockResultSet( List> rowsByIndex) { + this.rowsByIndex = rowsByIndex; + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + return new ResultSetMetaData() { + private final List columns = new ArrayList<>(rows.get(0).keySet()); + + @Override + public int getColumnCount() throws SQLException { + return columns.size(); + } + + @Override + public String getColumnName(int column) throws SQLException { + // JDBC columns are 1-based + return columns.get(column - 1); + } + + @Override + public int getColumnType(int column) throws SQLException { + Object value = rows.get(0).get(getColumnName(column)); + if (value instanceof String) return Types.VARCHAR; + if (value instanceof Integer) return Types.INTEGER; + if (value instanceof byte[]) return Types.BLOB; + // add more types as needed + return Types.OTHER; + } + + // Implement other methods as needed + @Override public boolean isAutoIncrement(int column) { return false; } + @Override public boolean isCaseSensitive(int column) { return true; } + @Override public boolean isSearchable(int column) { return true; } + @Override public boolean isCurrency(int column) { return false; } + @Override public int isNullable(int column) { return ResultSetMetaData.columnNullable; } + @Override public boolean isSigned(int column) { return true; } + @Override public int getColumnDisplaySize(int column) { return 255; } + @Override public String getColumnLabel(int column) throws SQLException { return getColumnName(column); } + @Override public String getSchemaName(int column) { return ""; } + @Override public int getPrecision(int column) { return 0; } + @Override public int getScale(int column) { return 0; } + @Override public String getTableName(int column) { return ""; } + @Override public String getCatalogName(int column) { return ""; } + @Override public String getColumnTypeName(int column) throws SQLException { return getColumnName(column); } + @Override public boolean isReadOnly(int column) { return true; } + @Override public boolean isWritable(int column) { return false; } + @Override public boolean isDefinitelyWritable(int column) { return false; } + @Override public String getColumnClassName(int column) { return Object.class.getName(); } + @Override public T unwrap(Class iface) { return null; } + @Override public boolean isWrapperFor(Class iface) { return false; } + }; + } + + @Override + public boolean next() throws SQLException { + if ((rows != null && cursor + 1 < rows.size()) || (rowsByIndex != null && cursor + 1 < rowsByIndex.size())) { + cursor++; + return true; + } + return false; + } + + @Override + public void close() throws SQLException { + + } + + @Override + public boolean wasNull() throws SQLException { + return false; + } + + @Override + public String getString(int columnIndex) throws SQLException { + return null; + } + + @Override + public boolean getBoolean(int columnIndex) throws SQLException { + return false; + } + + @Override + public byte getByte(int columnIndex) throws SQLException { + return 0; + } + + @Override + public short getShort(int columnIndex) throws SQLException { + return 0; + } + + @Override + public int getInt(int columnIndex) throws SQLException { + return 0; + } + + @Override + public long getLong(int columnIndex) throws SQLException { + return 0; + } + + @Override + public float getFloat(int columnIndex) throws SQLException { + return 0; + } + + @Override + public double getDouble(int columnIndex) throws SQLException { + return 0; + } + + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return null; + } + + @Override + public byte[] getBytes(int columnIndex) throws SQLException { + return new byte[0]; + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + return null; + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + return null; + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException { + return null; + } + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + return null; + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + return null; + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + return ((Blob) rowsByIndex.get(cursor).get(columnIndex - 1)).getBinaryStream(); + } + + @Override + public String getString(String columnLabel) throws SQLException { + return (String) rows.get(cursor).get(columnLabel); + } + + @Override + public boolean getBoolean(String columnLabel) throws SQLException { + return false; + } + + @Override + public byte getByte(String columnLabel) throws SQLException { + return 0; + } + + @Override + public short getShort(String columnLabel) throws SQLException { + return 0; + } + + @Override + public int getInt(String columnLabel) throws SQLException { + return (Integer) rows.get(cursor).get(columnLabel); + } + + @Override + public long getLong(String columnLabel) throws SQLException { + return 0; + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return 0; + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return 0; + } + + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return null; + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + return new byte[0]; + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + return null; + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return null; + } + + @Override + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return null; + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + return null; + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + return null; + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + return null; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return null; + } + + @Override + public void clearWarnings() throws SQLException { + + } + + @Override + public String getCursorName() throws SQLException { + return null; + } + + + @Override + public Object getObject(int columnIndex) throws SQLException { + return rowsByIndex.get(cursor).get(columnIndex - 1); + } + + @Override + public Object getObject(String columnLabel) throws SQLException { + return null; + } + + @Override + public int findColumn(String columnLabel) throws SQLException { + return 0; + } + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + return null; + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + return null; + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + return null; + } + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return null; + } + + @Override + public boolean isBeforeFirst() throws SQLException { + return false; + } + + @Override + public boolean isAfterLast() throws SQLException { + return false; + } + + @Override + public boolean isFirst() throws SQLException { + return false; + } + + @Override + public boolean isLast() throws SQLException { + return false; + } + + @Override + public void beforeFirst() throws SQLException { + + } + + @Override + public void afterLast() throws SQLException { + + } + + @Override + public boolean first() throws SQLException { + return false; + } + + @Override + public boolean last() throws SQLException { + return false; + } + + @Override + public int getRow() throws SQLException { + return 0; + } + + @Override + public boolean absolute(int row) throws SQLException { + return false; + } + + @Override + public boolean relative(int rows) throws SQLException { + return false; + } + + @Override + public boolean previous() throws SQLException { + return false; + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + + } + + @Override + public int getFetchDirection() throws SQLException { + return 0; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + + } + + @Override + public int getFetchSize() throws SQLException { + return 0; + } + + @Override + public int getType() throws SQLException { + return 0; + } + + @Override + public int getConcurrency() throws SQLException { + return 0; + } + + @Override + public boolean rowUpdated() throws SQLException { + return false; + } + + @Override + public boolean rowInserted() throws SQLException { + return false; + } + + @Override + public boolean rowDeleted() throws SQLException { + return false; + } + + @Override + public void updateNull(int columnIndex) throws SQLException { + + } + + @Override + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + + } + + @Override + public void updateByte(int columnIndex, byte x) throws SQLException { + + } + + @Override + public void updateShort(int columnIndex, short x) throws SQLException { + + } + + @Override + public void updateInt(int columnIndex, int x) throws SQLException { + + } + + @Override + public void updateLong(int columnIndex, long x) throws SQLException { + + } + + @Override + public void updateFloat(int columnIndex, float x) throws SQLException { + + } + + @Override + public void updateDouble(int columnIndex, double x) throws SQLException { + + } + + @Override + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + + } + + @Override + public void updateString(int columnIndex, String x) throws SQLException { + + } + + @Override + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + + } + + @Override + public void updateDate(int columnIndex, Date x) throws SQLException { + + } + + @Override + public void updateTime(int columnIndex, Time x) throws SQLException { + + } + + @Override + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { + + } + + @Override + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + + } + + @Override + public void updateObject(int columnIndex, Object x) throws SQLException { + + } + + @Override + public void updateNull(String columnLabel) throws SQLException { + + } + + @Override + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + + } + + @Override + public void updateByte(String columnLabel, byte x) throws SQLException { + + } + + @Override + public void updateShort(String columnLabel, short x) throws SQLException { + + } + + @Override + public void updateInt(String columnLabel, int x) throws SQLException { + + } + + @Override + public void updateLong(String columnLabel, long x) throws SQLException { + + } + + @Override + public void updateFloat(String columnLabel, float x) throws SQLException { + + } + + @Override + public void updateDouble(String columnLabel, double x) throws SQLException { + + } + + @Override + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + + } + + @Override + public void updateString(String columnLabel, String x) throws SQLException { + + } + + @Override + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + + } + + @Override + public void updateDate(String columnLabel, Date x) throws SQLException { + + } + + @Override + public void updateTime(String columnLabel, Time x) throws SQLException { + + } + + @Override + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + + } + + @Override + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + + } + + @Override + public void updateObject(String columnLabel, Object x) throws SQLException { + + } + + @Override + public void insertRow() throws SQLException { + + } + + @Override + public void updateRow() throws SQLException { + + } + + @Override + public void deleteRow() throws SQLException { + + } + + @Override + public void refreshRow() throws SQLException { + + } + + @Override + public void cancelRowUpdates() throws SQLException { + + } + + @Override + public void moveToInsertRow() throws SQLException { + + } + + @Override + public void moveToCurrentRow() throws SQLException { + + } + + @Override + public Statement getStatement() throws SQLException { + return null; + } + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + return null; + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + return null; + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + return (Blob) rowsByIndex.get(cursor).get(columnIndex - 1); + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + return null; + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + return null; + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + return null; + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + return null; + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + return null; + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + return null; + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + return null; + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + return null; + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + return null; + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + return null; + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + return null; + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + return null; + } + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + return null; + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + return null; + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + return null; + } + + @Override + public void updateRef(int columnIndex, Ref x) throws SQLException { + + } + + @Override + public void updateRef(String columnLabel, Ref x) throws SQLException { + + } + + @Override + public void updateBlob(int columnIndex, Blob x) throws SQLException { + + } + + @Override + public void updateBlob(String columnLabel, Blob x) throws SQLException { + + } + + @Override + public void updateClob(int columnIndex, Clob x) throws SQLException { + + } + + @Override + public void updateClob(String columnLabel, Clob x) throws SQLException { + + } + + @Override + public void updateArray(int columnIndex, Array x) throws SQLException { + + } + + @Override + public void updateArray(String columnLabel, Array x) throws SQLException { + + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + return null; + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + return null; + } + + @Override + public void updateRowId(int columnIndex, RowId x) throws SQLException { + + } + + @Override + public void updateRowId(String columnLabel, RowId x) throws SQLException { + + } + + @Override + public int getHoldability() throws SQLException { + return 0; + } + + @Override + public boolean isClosed() throws SQLException { + return false; + } + + @Override + public void updateNString(int columnIndex, String nString) throws SQLException { + + } + + @Override + public void updateNString(String columnLabel, String nString) throws SQLException { + + } + + @Override + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + + } + + @Override + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + return null; + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + return null; + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + return null; + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + return null; + } + + @Override + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + + } + + @Override + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + + } + + @Override + public String getNString(int columnIndex) throws SQLException { + return null; + } + + @Override + public String getNString(String columnLabel) throws SQLException { + return null; + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + return null; + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + return null; + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + + } + + @Override + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { + + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { + + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + + } + + @Override + public void updateClob(int columnIndex, Reader reader) throws SQLException { + + } + + @Override + public void updateClob(String columnLabel, Reader reader) throws SQLException { + + } + + @Override + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + + } + + @Override + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + return null; + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + return null; + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } + + + // Add more getters as needed... +} From 4d3d0e7b796402626fa60896d36f179701614ae9 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 12:06:51 -0400 Subject: [PATCH 32/57] Add test single blob --- pom.xml | 22 +++++++++++++++++-- .../ibm/mapepire/DataStreamProcessor.java | 11 +++++++++- .../requests/BlockRetrievableRequest.java | 4 +++- src/test/java/BlobTest.java | 4 ++-- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index fac51f1..4d2f54d 100644 --- a/pom.xml +++ b/pom.xml @@ -160,8 +160,8 @@ org.apache.maven.plugins maven-compiler-plugin - 8 - 8 + 9 + 9 @@ -227,6 +227,24 @@ websocket-server ${jetty.version} + + org.mockito + mockito-core + 5.11.0 + test + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test + diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 8597b4b..f49a3c6 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -274,7 +274,7 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final InputStream is, final String id, final String column) throws IOException { + public void sendResponse(final InputStream is, final String id, final String column, final int rowId) throws IOException { synchronized (s_replyWriterLock) { int curOffset = 0; byte[] buffer = new byte[8192]; @@ -292,6 +292,11 @@ public void sendResponse(final InputStream is, final String id, final String col System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); curOffset += idBytes.length; + // Copy rowId + buffer[curOffset] = (byte) rowId; + curOffset += 1; + + // Copy column length buffer[curOffset] = (byte) column.length(); curOffset += 1; @@ -300,6 +305,10 @@ public void sendResponse(final InputStream is, final String id, final String col System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); curOffset += columnNameBytes.length; + // Copy blob length + buffer[curOffset] = (byte) is.available(); + curOffset += 1; + int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); curOffset += bytesRead; diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 6985373..c5df4df 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -124,6 +124,8 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ final LinkedHashMap mapRowData = new LinkedHashMap(); final LinkedList terseRowData = new LinkedList(); final int numCols = _rs.getMetaData().getColumnCount(); + int rowId = i; + mapRowData.put("rowId", rowId); for (int col = 1; col <= numCols; ++col) { String column = _rs.getMetaData().getColumnName(col); Object cellData = _rs.getObject(col); @@ -141,7 +143,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Blob){ String id = this.getId(); InputStream is = _rs.getBinaryStream(col); - m_io.sendResponse(is, id, column); + m_io.sendResponse(is, id, column, rowId); // cellDataForResponse = _rs.getBytes(col); } diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 2b2a022..fe7dc81 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -121,7 +121,7 @@ public void testGetBinaryStreamByIndex() throws SQLException, IOException { } @Test - public void testBlockRetrievableReq() throws SQLException, IOException { + public void testSendingSingleBlob() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); BinarySender binarySender = mock(BinarySender.class); @@ -130,7 +130,7 @@ public void testBlockRetrievableReq() throws SQLException, IOException { BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); byte[] blobData = {1, 2, 3, 4, 5}; - byte[] expectedResult = {5, 49, 50, 51, 52, 53, 4, 98, 108, 111, 98, 1, 2, 3, 4, 5}; + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob Blob blob = new SerialBlob(blobData); // Use rowsByIndex to support getBinaryStream(int columnIndex) From f53069d159f99e85a7274940a9f530d0f37b349d Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 14:05:16 -0400 Subject: [PATCH 33/57] Add test multiple blobs in one row --- .../github/ibm/mapepire/BlobResponseData.java | 27 +++ .../ibm/mapepire/DataStreamProcessor.java | 81 +++++---- .../requests/BlockRetrievableRequest.java | 13 +- src/test/java/BlobTest.java | 160 +++++++++++++++--- 4 files changed, 221 insertions(+), 60 deletions(-) create mode 100644 src/main/java/com/github/ibm/mapepire/BlobResponseData.java diff --git a/src/main/java/com/github/ibm/mapepire/BlobResponseData.java b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java new file mode 100644 index 0000000..6264a21 --- /dev/null +++ b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java @@ -0,0 +1,27 @@ +package com.github.ibm.mapepire; + +import java.io.InputStream; + +public class BlobResponseData { + final InputStream is; + final String columnName; + final int rowId; + + public BlobResponseData(InputStream is, String columnName, int rowId){ + this.is = is; + this.columnName = columnName; + this.rowId = rowId; + } + + public InputStream getIs() { + return is; + } + + public String getColumnName(){ + return columnName; + } + + public int getRowId(){ + return rowId; + } +} diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index f49a3c6..ad0761e 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -14,6 +14,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; public class DataStreamProcessor implements Runnable { @@ -274,61 +275,73 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final InputStream is, final String id, final String column, final int rowId) throws IOException { + public void sendResponse(final String id, final List blobResponseDataArr) throws IOException { synchronized (s_replyWriterLock) { int curOffset = 0; byte[] buffer = new byte[8192]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); - byte[] columnNameBytes = column.getBytes(StandardCharsets.UTF_8); - if (idBytes.length > 255) { - throw new IllegalArgumentException("ID too long to encode in one byte length"); - } + for (int i = 0; i < blobResponseDataArr.size(); i++){ + buffer = new byte[8192]; + curOffset = 0; + BlobResponseData blobResponseData = blobResponseDataArr.get(i); + String columnName = blobResponseData.getColumnName(); + InputStream is = blobResponseData.getIs(); + int rowId = blobResponseData.getRowId(); - // First byte is the length of the ID - buffer[0] = (byte) idBytes.length; - curOffset += 1; - // Copy ID bytes after the length byte - System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); - curOffset += idBytes.length; + byte[] columnNameBytes = columnName.getBytes(StandardCharsets.UTF_8); - // Copy rowId - buffer[curOffset] = (byte) rowId; - curOffset += 1; + if (idBytes.length > 255) { + throw new IllegalArgumentException("ID too long to encode in one byte length"); + } + if (i == 0){ + // First byte is the length of the ID + buffer[curOffset] = (byte) idBytes.length; + curOffset += 1; + // Copy ID bytes after the length byte + System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); + curOffset += idBytes.length; + } - // Copy column length - buffer[curOffset] = (byte) column.length(); - curOffset += 1; + // Copy rowId + buffer[curOffset] = (byte) rowId; + curOffset += 1; - // copy column name - System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); - curOffset += columnNameBytes.length; - // Copy blob length - buffer[curOffset] = (byte) is.available(); - curOffset += 1; + // Copy column length + buffer[curOffset] = (byte) columnName.length(); + curOffset += 1; + // copy column name + System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); + curOffset += columnNameBytes.length; + + // Copy blob length + buffer[curOffset] = (byte) is.available(); + curOffset += 1; - int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); - curOffset += bytesRead; - if (bytesRead != -1) { - sendByteBuffer(buffer, curOffset, is); - } - while ((bytesRead = is.read(buffer)) != -1) { - sendByteBuffer(buffer, bytesRead, is); + int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); + curOffset += bytesRead; + if (bytesRead != -1) { + boolean isFinal = i == blobResponseDataArr.size() - 1 && is.available() == 0; + sendByteBuffer(buffer, curOffset, isFinal); + } + + while ((bytesRead = is.read(buffer)) != -1) { + boolean isFinal = i == blobResponseDataArr.size() - 1 && is.available() == 0; + sendByteBuffer(buffer, bytesRead, isFinal); + } } + } } - private void sendByteBuffer(byte[] buffer, int bytesRead, InputStream is) throws IOException { + private void sendByteBuffer(byte[] buffer, int bytesRead, boolean isFinal) throws IOException { // Wrap only the bytes actually read ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); - // Check if this is the last chunk - boolean isFinal = is.available() == 0; - m_binarySender.send(byteBuffer, isFinal); } diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index c5df4df..950eb30 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -4,17 +4,16 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.sql.*; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import com.github.ibm.mapepire.BlobResponseData; import com.github.ibm.mapepire.ClientRequest; import com.github.ibm.mapepire.DataStreamProcessor; import com.github.ibm.mapepire.SystemConnection; import com.google.gson.JsonObject; import com.ibm.as400.access.AS400JDBCBlobLocator; import com.ibm.as400.access.AS400JDBCParameterMetaData; +import com.github.ibm.mapepire.BlobResponseData; public abstract class BlockRetrievableRequest extends ClientRequest { @@ -110,6 +109,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ if (_rs.isClosed()) { return ret.setDone(true); } + final List blobResponseDataArray = new ArrayList<>(); for (int i = 0; i < _numRows; ++i) { if (!_rs.next()) { ret.setDone(true); @@ -143,8 +143,8 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Blob){ String id = this.getId(); InputStream is = _rs.getBinaryStream(col); - m_io.sendResponse(is, id, column, rowId); - + BlobResponseData blobResponseData = new BlobResponseData(is, column, rowId); + blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } else { @@ -158,6 +158,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } ret.add(_isTerseDataFormat ? terseRowData : mapRowData); } + m_io.sendResponse(this.getId(), blobResponseDataArray); return ret; } diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index fe7dc81..e8670ed 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -100,28 +100,28 @@ void testBlobColumnInResultSet() throws Exception { assertFalse(rs.next()); } - @Test - public void testGetBinaryStreamByIndex() throws SQLException, IOException { - // Prepare blob data - byte[] blobData = {1, 2, 3, 4, 5}; - - // Use rowsByIndex to support getBinaryStream(int columnIndex) - List> rowsByIndex = new ArrayList<>(); - rowsByIndex.add(Arrays.asList("Alice", new ByteArrayInputStream(blobData))); - - MockResultSet rs = new MockResultSet(rowsByIndex); - - assertTrue(rs.next(), "Should have first row"); - - InputStream is = rs.getBinaryStream(2); // second column - assertNotNull(is, "BinaryStream should not be null"); - - byte[] readData = is.readAllBytes(); - assertArrayEquals(blobData, readData, "Binary data should match"); - } +// @Test +// public void testGetBinaryStreamByIndex() throws SQLException, IOException { +// // Prepare blob data +// byte[] blobData = {1, 2, 3, 4, 5}; +// +// // Use rowsByIndex to support getBinaryStream(int columnIndex) +// List> rowsByIndex = new ArrayList<>(); +// rowsByIndex.add(Arrays.asList("Alice", new ByteArrayInputStream(blobData))); +// +// MockResultSet rs = new MockResultSet(rowsByIndex); +// +// assertTrue(rs.next(), "Should have first row"); +// +// InputStream is = rs.getBinaryStream(2); // second column +// assertNotNull(is, "BinaryStream should not be null"); +// +// byte[] readData = is.readAllBytes(); +// assertArrayEquals(blobData, readData, "Binary data should match"); +// } @Test - public void testSendingSingleBlob() throws SQLException, IOException { + public void testSendingSingleRowWithBlob() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); BinarySender binarySender = mock(BinarySender.class); @@ -157,7 +157,127 @@ public void testSendingSingleBlob() throws SQLException, IOException { byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); assertArrayEquals(expectedResult, actual); assertTrue(firstFlag); + } + + @Test + public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "12345"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blob1Data = {1, 2, 3, 4, 5}; + byte[] blob2Data = {6, 7, 8}; + + byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult2 = {1, 4, 98, 108, 111, 98, 3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob + + Blob blob1 = new SerialBlob(blob1Data); + Blob blob2 = new SerialBlob(blob2Data); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob1)); + rowsByIndex.add(Arrays.asList(blob2)); + + Map row1 = new HashMap<>(); + Map row2 = new HashMap<>(); + + row1.put("blob", blob1); + row2.put("blob", blob2); + + List> rows = Arrays.asList(row1, row2); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 2, false); + + verify(binarySender, times(2)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); + // Verify first call + ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); + Boolean firstFlag = booleanCaptor.getAllValues().get(0); + byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); + assertArrayEquals(expectedResult1, actual); + assertFalse(firstFlag); + + // Verify second call + ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); + Boolean secondFlag = booleanCaptor.getAllValues().get(1); + byte[] actualSecond = Arrays.copyOfRange(secondBuf.array(), secondBuf.position(), secondBuf.limit()); + assertArrayEquals(expectedResult2, actualSecond); + assertTrue(secondFlag); + } + + + @Test + public void testSingleRowMultipleBlobs() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "123"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blobData1 = {10, 11}; + byte[] blobData2 = {20, 21, 22}; + Blob blob1 = new SerialBlob(blobData1); + Blob blob2 = new SerialBlob(blobData2); + + + + byte[] expected1 = { + 3, '1','2','3', // queryId + 0, // rowId + 1, 'a', // colName "a" + 2, 10, 11, // blob + }; + + byte[] expected2 = { + 0, // rowId + 2, 'b','b', // colName "bb" + 3, 20, 21, 22 // blob + }; + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob1, blob2)); + + Map row1 = new HashMap<>(); + + row1.put("a", blob1); + row1.put("bb", blob2); + + List> rows = Arrays.asList(row1); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 1, false); + + verify(binarySender, times(2)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); + + // Verify first call + ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); + Boolean firstFlag = booleanCaptor.getAllValues().get(0); + byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); + assertArrayEquals(expected1, actual); + assertFalse(firstFlag); + + // Verify second call + ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); + Boolean secondFlag = booleanCaptor.getAllValues().get(1); + byte[] actualSecond = Arrays.copyOfRange(secondBuf.array(), secondBuf.position(), secondBuf.limit()); + assertArrayEquals(expected2, actualSecond); + assertTrue(secondFlag); } From bcae6c2479495acf9bc505834db437e88bc73458 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 16:47:54 -0400 Subject: [PATCH 34/57] Add more tests --- src/test/java/BlobTest.java | 130 +++++++++++++++++++++++++++++++ src/test/java/MockResultSet.java | 3 + 2 files changed, 133 insertions(+) diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index e8670ed..8e7b682 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -280,5 +280,135 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { assertTrue(secondFlag); } + @Test + public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "query7"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blob1Data = {1, 2, 3, 4, 5}; + byte[] blob2Data = {6, 7, 8}; + + byte[] expected1 = { + 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId + 0, // rowId + 11, 'b', 'l','o','b','C','o','l','u','m','n','1', // colName "a" + 5, 1, 2, 3, 4 ,5 // blob + }; + + byte[] expected2 = { + 0, // rowId + 11, 'b', 'l','o','b','C','o','l','u','m','n','2', + 3, 6, 7, 8 // blob + }; + + byte[] expected3 = { + 1, // rowId + 11, 'b', 'l','o','b','C','o','l','u','m','n','1', + 5, 1, 2, 3, 4 ,5 // blob + }; + + byte[] expected4 = { + 1, // rowId + 11, 'b', 'l','o','b','C','o','l','u','m','n','2', + 3, 6, 7, 8 // blob + }; + + + Blob blob1 = new SerialBlob(blob1Data); + Blob blob2 = new SerialBlob(blob2Data); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob1, blob2)); + rowsByIndex.add(Arrays.asList(blob1, blob2)); + + Map row1 = new HashMap<>(); + Map row2 = new HashMap<>(); + + row1.put("blobColumn1", blob1); + row1.put("blobColumn2", blob2); + + row2.put("blobColumn1", blob1); + row2.put("blobColumn2", blob2); + + + List> rows = Arrays.asList(row1, row2); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 2, false); + + verify(binarySender, times(4)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(4)).send(bufferCaptor.capture(), booleanCaptor.capture()); + + // Verify first call + ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); + Boolean firstFlag = booleanCaptor.getAllValues().get(0); + byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); + assertArrayEquals(expected1, actual); + assertFalse(firstFlag); + + // Verify second call + ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); + Boolean secondFlag = booleanCaptor.getAllValues().get(1); + byte[] actualSecond = Arrays.copyOfRange(secondBuf.array(), secondBuf.position(), secondBuf.limit()); + assertArrayEquals(expected2, actualSecond); + assertFalse(secondFlag); + + // Verify third call + ByteBuffer thirdBuf = bufferCaptor.getAllValues().get(2); + Boolean thirdFlag = booleanCaptor.getAllValues().get(2); + byte[] actualThird = Arrays.copyOfRange(thirdBuf.array(), thirdBuf.position(), thirdBuf.limit()); + assertArrayEquals(expected3, actualThird); + assertFalse(thirdFlag); + + // Verify fourth call + ByteBuffer fourthBuf = bufferCaptor.getAllValues().get(3); + Boolean fourthFlag = booleanCaptor.getAllValues().get(3); + byte[] actualFourth = Arrays.copyOfRange(fourthBuf.array(), fourthBuf.position(), fourthBuf.limit()); + assertArrayEquals(expected4, actualFourth); + assertTrue(fourthFlag); + } + + + @Test + public void testSendingSingleRowWithEmptyBlob() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "12345"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blobData = {}; + Blob blob = new SerialBlob(blobData); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob)); + + Map row1 = new HashMap<>(); + row1.put("blob", blob); + List> rows = Arrays.asList(row1); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 1, false); + + verify(binarySender, times(0)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(0)).send(bufferCaptor.capture(), booleanCaptor.capture()); + + } + } diff --git a/src/test/java/MockResultSet.java b/src/test/java/MockResultSet.java index d5f4d57..ca68e73 100644 --- a/src/test/java/MockResultSet.java +++ b/src/test/java/MockResultSet.java @@ -26,6 +26,9 @@ public MockResultSet( List> rowsByIndex) { public ResultSetMetaData getMetaData() throws SQLException { return new ResultSetMetaData() { private final List columns = new ArrayList<>(rows.get(0).keySet()); + { // instance initializer + Collections.sort(columns); + } @Override public int getColumnCount() throws SQLException { From 1bb4e5ed23ac813d5c80a66d6d4877513687188d Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 19:57:05 -0400 Subject: [PATCH 35/57] change blob length to 4 bytes --- .../ibm/mapepire/DataStreamProcessor.java | 10 +- src/test/java/BlobTest.java | 139 ++++++++++++++++-- 2 files changed, 137 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index ad0761e..6bfdcdc 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -318,9 +318,13 @@ public void sendResponse(final String id, final List blobRespo curOffset += columnNameBytes.length; // Copy blob length - buffer[curOffset] = (byte) is.available(); - curOffset += 1; - + ByteBuffer blobLength = ByteBuffer.allocate(4); + blobLength.putInt(is.available()); // default is big-endian + byte[] bytes = blobLength.array(); + for (int j = 0; j < 4; j++){ + buffer[curOffset] = bytes[j]; + curOffset += 1; + } int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); curOffset += bytesRead; diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 8e7b682..3f49f84 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -130,7 +130,7 @@ public void testSendingSingleRowWithBlob() throws SQLException, IOException { BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); byte[] blobData = {1, 2, 3, 4, 5}; - byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob Blob blob = new SerialBlob(blobData); // Use rowsByIndex to support getBinaryStream(int columnIndex) @@ -171,8 +171,8 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException byte[] blob1Data = {1, 2, 3, 4, 5}; byte[] blob2Data = {6, 7, 8}; - byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob - byte[] expectedResult2 = {1, 4, 98, 108, 111, 98, 3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult2 = {1, 4, 98, 108, 111, 98, 0,0,0,3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob Blob blob1 = new SerialBlob(blob1Data); Blob blob2 = new SerialBlob(blob2Data); @@ -236,13 +236,13 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { 3, '1','2','3', // queryId 0, // rowId 1, 'a', // colName "a" - 2, 10, 11, // blob + 0,0,0,2, 10, 11, // blob }; byte[] expected2 = { 0, // rowId 2, 'b','b', // colName "bb" - 3, 20, 21, 22 // blob + 0,0,0,3, 20, 21, 22 // blob }; // Use rowsByIndex to support getBinaryStream(int columnIndex) List> rowsByIndex = new ArrayList<>(); @@ -296,25 +296,25 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId 0, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','1', // colName "a" - 5, 1, 2, 3, 4 ,5 // blob + 0,0,0,5, 1, 2, 3, 4 ,5 // blob }; byte[] expected2 = { 0, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', - 3, 6, 7, 8 // blob + 0,0,0,3, 6, 7, 8 // blob }; byte[] expected3 = { 1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','1', - 5, 1, 2, 3, 4 ,5 // blob + 0,0,0,5, 1, 2, 3, 4 ,5 // blob }; byte[] expected4 = { 1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', - 3, 6, 7, 8 // blob + 0,0,0,3, 6, 7, 8 // blob }; @@ -410,5 +410,126 @@ public void testSendingSingleRowWithEmptyBlob() throws SQLException, IOException } + @Test + public void testSendingSingleRowWithBlobandNonblobColumns() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "12345"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + byte[] blobData = {1, 2, 3, 4, 5}; + String strCol = "Some string column data"; + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + Blob blob = new SerialBlob(blobData); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + + rowsByIndex.add(Arrays.asList(blob, strCol)); + + Map row1 = new HashMap<>(); + row1.put("blob", blob); + row1.put("strCol", strCol); + + List> rows = Arrays.asList(row1); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); + block.getNextDataBlockUsage(rs, 1, false); + + verify(binarySender, times(1)).send(any(ByteBuffer.class), anyBoolean()); + + // Assert - capture arguments + ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); + + verify(binarySender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); + + // Verify first call + ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); + Boolean firstFlag = booleanCaptor.getAllValues().get(0); + byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); + assertArrayEquals(expectedResult, actual); + assertTrue(firstFlag); + } + + + @Test + public void testSendingSingleRowWithLongBlob() throws SQLException, IOException { + JsonObject obj = new JsonObject(); + obj.addProperty("id", "12345"); + BinarySender binarySender = mock(BinarySender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + + + BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); + int blobSize = 5 * 1024 * 1024; // 5 MB + byte[] blobData = new byte[blobSize]; + for (int i = 0; i < blobSize; i++) { + blobData[i] = (byte) (i % 256); // repeat 0-255 + } + +// byte[] expected1 = { +// 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId +// 0, // rowId +// 11, 'b', 'l','o','b','C','o','l','u','m','n','1', // colName "a" +// 5, 1, 2, 3, 4 ,5 // blob +// }; + byte[] expectedResult = { + 5, 49, 50, 51, 52, 53, // querylength, id + 0, // row id + 4, 98, 108, 111, 98, // col name length, colname + 0, 80, 0, 0, 0, 1, 2, 3, 4, 5}; // blob length, blob + Blob blob = new SerialBlob(blobData); + + // Use rowsByIndex to support getBinaryStream(int columnIndex) + List> rowsByIndex = new ArrayList<>(); + rowsByIndex.add(Arrays.asList(blob)); + + Map row1 = new HashMap<>(); + row1.put("blob", blob); + List> rows = Arrays.asList(row1); + MockResultSet rs = new MockResultSet(rows, rowsByIndex); +// +// verify(binarySender, times(641)).send(any(ByteBuffer.class), anyBoolean()); +// +// // Assert - capture arguments +// ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); +// ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); +// +// verify(binarySender, times(641)).send(bufferCaptor.capture(), booleanCaptor.capture()); +// +// // Verify first call +// ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); +// Boolean firstFlag = booleanCaptor.getAllValues().get(0); +// byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), expectedResult.length); +// assertArrayEquals(expectedResult, actual); +// assertTrue(firstFlag); + + List capturedBuffers = new ArrayList<>(); + List capturedFlags = new ArrayList<>(); + + doAnswer(invocation -> { + ByteBuffer buf = invocation.getArgument(0); + boolean isFinal = invocation.getArgument(1); + + // Copy the bytes immediately, so later modifications don't affect this snapshot + byte[] snapshot = Arrays.copyOfRange(buf.array(), buf.position(), buf.limit()); + capturedBuffers.add(snapshot); + capturedFlags.add(isFinal); + + return null; + }).when(binarySender).send(any(ByteBuffer.class), anyBoolean()); + block.getNextDataBlockUsage(rs, 1, false); + + byte[] actualFirst21 = Arrays.copyOfRange(capturedBuffers.get(0), 0, 22); + + assertArrayEquals(expectedResult, actualFirst21); + assertFalse(capturedFlags.get(0)); + + + + } + } From f26ca513155836a43f567bd110c68a0d5c0de09a Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 20:20:06 -0400 Subject: [PATCH 36/57] Made row id 4 bytes --- .../ibm/mapepire/DataStreamProcessor.java | 10 +++++--- src/test/java/BlobTest.java | 24 +++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 6bfdcdc..024b675 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -305,9 +305,13 @@ public void sendResponse(final String id, final List blobRespo } // Copy rowId - buffer[curOffset] = (byte) rowId; - curOffset += 1; - + ByteBuffer rowIdBuffer = ByteBuffer.allocate(4); + rowIdBuffer.putInt(rowId); // default is big-endian + byte[] rowIdBytes = rowIdBuffer.array(); + for (int j = 0; j < 4; j++){ + buffer[curOffset] = rowIdBytes[j]; + curOffset += 1; + } // Copy column length buffer[curOffset] = (byte) columnName.length(); diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 3f49f84..1cc5e09 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -130,7 +130,7 @@ public void testSendingSingleRowWithBlob() throws SQLException, IOException { BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); byte[] blobData = {1, 2, 3, 4, 5}; - byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0,0,0,0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob Blob blob = new SerialBlob(blobData); // Use rowsByIndex to support getBinaryStream(int columnIndex) @@ -171,8 +171,8 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException byte[] blob1Data = {1, 2, 3, 4, 5}; byte[] blob2Data = {6, 7, 8}; - byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob - byte[] expectedResult2 = {1, 4, 98, 108, 111, 98, 0,0,0,3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0,0,0,0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult2 = {0,0,0,1, 4, 98, 108, 111, 98, 0,0,0,3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob Blob blob1 = new SerialBlob(blob1Data); Blob blob2 = new SerialBlob(blob2Data); @@ -234,13 +234,13 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { byte[] expected1 = { 3, '1','2','3', // queryId - 0, // rowId + 0,0,0,0, // rowId 1, 'a', // colName "a" 0,0,0,2, 10, 11, // blob }; byte[] expected2 = { - 0, // rowId + 0,0,0,0, // rowId 2, 'b','b', // colName "bb" 0,0,0,3, 20, 21, 22 // blob }; @@ -294,25 +294,25 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti byte[] expected1 = { 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId - 0, // rowId + 0,0,0,0, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','1', // colName "a" 0,0,0,5, 1, 2, 3, 4 ,5 // blob }; byte[] expected2 = { - 0, // rowId + 0,0,0,0, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', 0,0,0,3, 6, 7, 8 // blob }; byte[] expected3 = { - 1, // rowId + 0,0,0,1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','1', 0,0,0,5, 1, 2, 3, 4 ,5 // blob }; byte[] expected4 = { - 1, // rowId + 0,0,0,1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', 0,0,0,3, 6, 7, 8 // blob }; @@ -421,7 +421,7 @@ public void testSendingSingleRowWithBlobandNonblobColumns() throws SQLException, BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); byte[] blobData = {1, 2, 3, 4, 5}; String strCol = "Some string column data"; - byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult = {5, 49, 50, 51, 52, 53, 0,0,0,0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob Blob blob = new SerialBlob(blobData); // Use rowsByIndex to support getBinaryStream(int columnIndex) @@ -477,7 +477,7 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException // }; byte[] expectedResult = { 5, 49, 50, 51, 52, 53, // querylength, id - 0, // row id + 0,0,0,0, // row id 4, 98, 108, 111, 98, // col name length, colname 0, 80, 0, 0, 0, 1, 2, 3, 4, 5}; // blob length, blob Blob blob = new SerialBlob(blobData); @@ -522,7 +522,7 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException }).when(binarySender).send(any(ByteBuffer.class), anyBoolean()); block.getNextDataBlockUsage(rs, 1, false); - byte[] actualFirst21 = Arrays.copyOfRange(capturedBuffers.get(0), 0, 22); + byte[] actualFirst21 = Arrays.copyOfRange(capturedBuffers.get(0), 0, 25); assertArrayEquals(expectedResult, actualFirst21); assertFalse(capturedFlags.get(0)); From a066ec012bda53c710a56746c61ddbbee4cde523 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 18 Aug 2025 20:30:55 -0400 Subject: [PATCH 37/57] test second buf of long blob --- src/test/java/BlobTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 1cc5e09..6860c43 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -480,6 +480,9 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException 0,0,0,0, // row id 4, 98, 108, 111, 98, // col name length, colname 0, 80, 0, 0, 0, 1, 2, 3, 4, 5}; // blob length, blob + + byte[] expectedResult2 = { + -19, -18, -17, -16, -15}; // blob length, blob Blob blob = new SerialBlob(blobData); // Use rowsByIndex to support getBinaryStream(int columnIndex) @@ -523,8 +526,12 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException block.getNextDataBlockUsage(rs, 1, false); byte[] actualFirst21 = Arrays.copyOfRange(capturedBuffers.get(0), 0, 25); + byte[] actualSecondBuf = Arrays.copyOfRange(capturedBuffers.get(1), 0, 5); + assertArrayEquals(expectedResult, actualFirst21); + assertArrayEquals(expectedResult2, actualSecondBuf); + assertFalse(capturedFlags.get(0)); From 8934795790e5d2666c620ae0d708904fbc589349 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 20 Aug 2025 22:31:52 -0400 Subject: [PATCH 38/57] dont use is.isavailable --- .../java/com/github/ibm/mapepire/BlobResponseData.java | 8 +++++++- .../java/com/github/ibm/mapepire/DataStreamProcessor.java | 8 +++++--- .../ibm/mapepire/requests/BlockRetrievableRequest.java | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/BlobResponseData.java b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java index 6264a21..cf08fa3 100644 --- a/src/main/java/com/github/ibm/mapepire/BlobResponseData.java +++ b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java @@ -6,11 +6,13 @@ public class BlobResponseData { final InputStream is; final String columnName; final int rowId; + final int length; - public BlobResponseData(InputStream is, String columnName, int rowId){ + public BlobResponseData(InputStream is, String columnName, int rowId, int length){ this.is = is; this.columnName = columnName; this.rowId = rowId; + this.length = length; } public InputStream getIs() { @@ -24,4 +26,8 @@ public String getColumnName(){ public int getRowId(){ return rowId; } + + public int getLength(){ + return length; + } } diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 024b675..856c8c8 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -323,7 +323,7 @@ public void sendResponse(final String id, final List blobRespo // Copy blob length ByteBuffer blobLength = ByteBuffer.allocate(4); - blobLength.putInt(is.available()); // default is big-endian + blobLength.putInt(blobResponseData.getLength()); // default is big-endian byte[] bytes = blobLength.array(); for (int j = 0; j < 4; j++){ buffer[curOffset] = bytes[j]; @@ -332,13 +332,15 @@ public void sendResponse(final String id, final List blobRespo int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); curOffset += bytesRead; + int totalBytesRead = bytesRead; if (bytesRead != -1) { - boolean isFinal = i == blobResponseDataArr.size() - 1 && is.available() == 0; + boolean isFinal = i == blobResponseDataArr.size() - 1 && totalBytesRead == blobResponseData.getLength(); sendByteBuffer(buffer, curOffset, isFinal); } while ((bytesRead = is.read(buffer)) != -1) { - boolean isFinal = i == blobResponseDataArr.size() - 1 && is.available() == 0; + totalBytesRead += bytesRead; + boolean isFinal = i == blobResponseDataArr.size() - 1 && totalBytesRead == blobResponseData.getLength(); sendByteBuffer(buffer, bytesRead, isFinal); } } diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 950eb30..9db5949 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -142,8 +142,9 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ cellDataForResponse = cellData; } else if (cellData instanceof Blob){ String id = this.getId(); + int blobLength = (int) ((Blob) cellData).length(); InputStream is = _rs.getBinaryStream(col); - BlobResponseData blobResponseData = new BlobResponseData(is, column, rowId); + BlobResponseData blobResponseData = new BlobResponseData(is, column, rowId, blobLength); blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } From 9ac009c4a5e830ef81966b198bd6a82098544846 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 20 Aug 2025 22:47:41 -0400 Subject: [PATCH 39/57] store blob locator --- .../com/github/ibm/mapepire/BlobResponseData.java | 11 ++++++----- .../com/github/ibm/mapepire/DataStreamProcessor.java | 4 ++-- .../mapepire/requests/BlockRetrievableRequest.java | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/BlobResponseData.java b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java index cf08fa3..a53f4f6 100644 --- a/src/main/java/com/github/ibm/mapepire/BlobResponseData.java +++ b/src/main/java/com/github/ibm/mapepire/BlobResponseData.java @@ -1,22 +1,23 @@ package com.github.ibm.mapepire; import java.io.InputStream; +import java.sql.Blob; public class BlobResponseData { - final InputStream is; + final Blob blob; final String columnName; final int rowId; final int length; - public BlobResponseData(InputStream is, String columnName, int rowId, int length){ - this.is = is; + public BlobResponseData(Blob is, String columnName, int rowId, int length){ + this.blob = is; this.columnName = columnName; this.rowId = rowId; this.length = length; } - public InputStream getIs() { - return is; + public Blob getBlob() { + return blob; } public String getColumnName(){ diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 856c8c8..e2ebc97 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -275,7 +275,7 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final String id, final List blobResponseDataArr) throws IOException { + public void sendResponse(final String id, final List blobResponseDataArr) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; byte[] buffer = new byte[8192]; @@ -286,7 +286,7 @@ public void sendResponse(final String id, final List blobRespo curOffset = 0; BlobResponseData blobResponseData = blobResponseDataArr.get(i); String columnName = blobResponseData.getColumnName(); - InputStream is = blobResponseData.getIs(); + InputStream is = blobResponseData.getBlob().getBinaryStream(); int rowId = blobResponseData.getRowId(); byte[] columnNameBytes = columnName.getBytes(StandardCharsets.UTF_8); diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 9db5949..fe306c1 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -143,8 +143,8 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Blob){ String id = this.getId(); int blobLength = (int) ((Blob) cellData).length(); - InputStream is = _rs.getBinaryStream(col); - BlobResponseData blobResponseData = new BlobResponseData(is, column, rowId, blobLength); +// InputStream is = _rs.getBinaryStream(col); + BlobResponseData blobResponseData = new BlobResponseData((Blob)cellData, column, rowId, blobLength); blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } From 7c9a5f4f971423bc710b4d750502df5540406043 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Wed, 20 Aug 2025 23:12:03 -0400 Subject: [PATCH 40/57] Upgrade jt400 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4d2f54d..f6a6fe2 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ net.sf.jt400 jt400 - 11.2 + 21.0.5 provided From 6b22a0bb57e103fa2851c64c4f375405665bf9b3 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 21 Aug 2025 09:34:35 -0400 Subject: [PATCH 41/57] send blobs individually --- .../github/ibm/mapepire/DataStreamProcessor.java | 13 +++++-------- .../requests/BlockRetrievableRequest.java | 11 ++++++++--- src/test/java/BlobTest.java | 16 ++++++++++------ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index e2ebc97..3eb1cbc 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -275,16 +275,14 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep // } // } - public void sendResponse(final String id, final List blobResponseDataArr) throws IOException, SQLException { + public void sendResponse(final String id, final BlobResponseData blobResponseData) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; byte[] buffer = new byte[8192]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); - for (int i = 0; i < blobResponseDataArr.size(); i++){ buffer = new byte[8192]; curOffset = 0; - BlobResponseData blobResponseData = blobResponseDataArr.get(i); String columnName = blobResponseData.getColumnName(); InputStream is = blobResponseData.getBlob().getBinaryStream(); int rowId = blobResponseData.getRowId(); @@ -295,14 +293,13 @@ public void sendResponse(final String id, final List blobRespo throw new IllegalArgumentException("ID too long to encode in one byte length"); } - if (i == 0){ // First byte is the length of the ID buffer[curOffset] = (byte) idBytes.length; curOffset += 1; // Copy ID bytes after the length byte System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); curOffset += idBytes.length; - } + // Copy rowId ByteBuffer rowIdBuffer = ByteBuffer.allocate(4); @@ -334,16 +331,16 @@ public void sendResponse(final String id, final List blobRespo curOffset += bytesRead; int totalBytesRead = bytesRead; if (bytesRead != -1) { - boolean isFinal = i == blobResponseDataArr.size() - 1 && totalBytesRead == blobResponseData.getLength(); + boolean isFinal = totalBytesRead == blobResponseData.getLength(); sendByteBuffer(buffer, curOffset, isFinal); } while ((bytesRead = is.read(buffer)) != -1) { totalBytesRead += bytesRead; - boolean isFinal = i == blobResponseDataArr.size() - 1 && totalBytesRead == blobResponseData.getLength(); + boolean isFinal = totalBytesRead == blobResponseData.getLength(); sendByteBuffer(buffer, bytesRead, isFinal); } - } + } } diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index fe306c1..3e91680 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -124,6 +124,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ final LinkedHashMap mapRowData = new LinkedHashMap(); final LinkedList terseRowData = new LinkedList(); final int numCols = _rs.getMetaData().getColumnCount(); + int blobsNeeded = 0; int rowId = i; mapRowData.put("rowId", rowId); for (int col = 1; col <= numCols; ++col) { @@ -141,11 +142,12 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Number || cellData instanceof Boolean) { cellDataForResponse = cellData; } else if (cellData instanceof Blob){ - String id = this.getId(); + blobsNeeded++; int blobLength = (int) ((Blob) cellData).length(); // InputStream is = _rs.getBinaryStream(col); BlobResponseData blobResponseData = new BlobResponseData((Blob)cellData, column, rowId, blobLength); - blobResponseDataArray.add(blobResponseData); + m_io.sendResponse(this.getId(), blobResponseData); +// blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } else { @@ -158,8 +160,11 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } } ret.add(_isTerseDataFormat ? terseRowData : mapRowData); + Map blobsNeededMap = new HashMap(); + blobsNeededMap.put("blobsNeeded", blobsNeeded); + ret.add(blobsNeededMap); } - m_io.sendResponse(this.getId(), blobResponseDataArray); +// m_io.sendResponse(this.getId(), blobResponseDataArray); return ret; } diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 6860c43..6c13ff0 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -172,7 +172,7 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException byte[] blob2Data = {6, 7, 8}; byte[] expectedResult1 = {5, 49, 50, 51, 52, 53, 0,0,0,0, 4, 98, 108, 111, 98, 0,0,0,5, 1, 2, 3, 4, 5}; // queryIdLength | queryId | rowId | colNameLength | colName| bloblLength| blob - byte[] expectedResult2 = {0,0,0,1, 4, 98, 108, 111, 98, 0,0,0,3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob + byte[] expectedResult2 = {5, 49, 50, 51, 52, 53, 0,0,0,1, 4, 98, 108, 111, 98, 0,0,0,3, 6, 7, 8}; // rowId | colNameLength | colName| bloblLength| blob Blob blob1 = new SerialBlob(blob1Data); Blob blob2 = new SerialBlob(blob2Data); @@ -205,7 +205,7 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException Boolean firstFlag = booleanCaptor.getAllValues().get(0); byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); assertArrayEquals(expectedResult1, actual); - assertFalse(firstFlag); + assertTrue(firstFlag); // Verify second call ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); @@ -240,6 +240,7 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { }; byte[] expected2 = { + 3, '1','2','3', // queryId 0,0,0,0, // rowId 2, 'b','b', // colName "bb" 0,0,0,3, 20, 21, 22 // blob @@ -270,7 +271,7 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { Boolean firstFlag = booleanCaptor.getAllValues().get(0); byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); assertArrayEquals(expected1, actual); - assertFalse(firstFlag); + assertTrue(firstFlag); // Verify second call ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); @@ -300,18 +301,21 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti }; byte[] expected2 = { + 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId 0,0,0,0, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', 0,0,0,3, 6, 7, 8 // blob }; byte[] expected3 = { + 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId 0,0,0,1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','1', 0,0,0,5, 1, 2, 3, 4 ,5 // blob }; byte[] expected4 = { + 6, 'q', 'u', 'e', 'r', 'y', '7', // queryId 0,0,0,1, // rowId 11, 'b', 'l','o','b','C','o','l','u','m','n','2', 0,0,0,3, 6, 7, 8 // blob @@ -353,21 +357,21 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti Boolean firstFlag = booleanCaptor.getAllValues().get(0); byte[] actual = Arrays.copyOfRange(firstBuf.array(), firstBuf.position(), firstBuf.limit()); assertArrayEquals(expected1, actual); - assertFalse(firstFlag); + assertTrue(firstFlag); // Verify second call ByteBuffer secondBuf = bufferCaptor.getAllValues().get(1); Boolean secondFlag = booleanCaptor.getAllValues().get(1); byte[] actualSecond = Arrays.copyOfRange(secondBuf.array(), secondBuf.position(), secondBuf.limit()); assertArrayEquals(expected2, actualSecond); - assertFalse(secondFlag); + assertTrue(secondFlag); // Verify third call ByteBuffer thirdBuf = bufferCaptor.getAllValues().get(2); Boolean thirdFlag = booleanCaptor.getAllValues().get(2); byte[] actualThird = Arrays.copyOfRange(thirdBuf.array(), thirdBuf.position(), thirdBuf.limit()); assertArrayEquals(expected3, actualThird); - assertFalse(thirdFlag); + assertTrue(thirdFlag); // Verify fourth call ByteBuffer fourthBuf = bufferCaptor.getAllValues().get(3); From 491d4ac095552b09ec824d4ac4129ff415e2f483 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 21 Aug 2025 09:46:07 -0400 Subject: [PATCH 42/57] Bump buffer size to 4mb --- .../java/com/github/ibm/mapepire/DataStreamProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 3eb1cbc..1bfb02f 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -278,10 +278,10 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep public void sendResponse(final String id, final BlobResponseData blobResponseData) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; - byte[] buffer = new byte[8192]; + byte[] buffer = new byte[4 * 1024 * 1024]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); - buffer = new byte[8192]; +// buffer = new byte[8192]; curOffset = 0; String columnName = blobResponseData.getColumnName(); InputStream is = blobResponseData.getBlob().getBinaryStream(); From 70e65e07894731636d281c4a671bc2f978be193e Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Thu, 21 Aug 2025 10:07:44 -0400 Subject: [PATCH 43/57] Move blobs needed out of rows --- .../github/ibm/mapepire/requests/BlockRetrievableRequest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 3e91680..4e9a983 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -160,9 +160,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } } ret.add(_isTerseDataFormat ? terseRowData : mapRowData); - Map blobsNeededMap = new HashMap(); - blobsNeededMap.put("blobsNeeded", blobsNeeded); - ret.add(blobsNeededMap); + addReplyData("blobsNeeded", blobsNeeded); } // m_io.sendResponse(this.getId(), blobResponseDataArray); return ret; From 6eaff738bc923b4b48e043559a14f3538252f728 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Fri, 22 Aug 2025 13:54:53 -0400 Subject: [PATCH 44/57] change logic for inserting blobs --- .../github/ibm/mapepire/BlobRequestData.java | 26 ++++++ .../ibm/mapepire/DataStreamProcessor.java | 32 ++++--- .../ibm/mapepire/requests/PrepareSql.java | 10 +++ .../github/ibm/mapepire/requests/RunBlob.java | 83 +++---------------- 4 files changed, 66 insertions(+), 85 deletions(-) create mode 100644 src/main/java/com/github/ibm/mapepire/BlobRequestData.java diff --git a/src/main/java/com/github/ibm/mapepire/BlobRequestData.java b/src/main/java/com/github/ibm/mapepire/BlobRequestData.java new file mode 100644 index 0000000..360eca3 --- /dev/null +++ b/src/main/java/com/github/ibm/mapepire/BlobRequestData.java @@ -0,0 +1,26 @@ +package com.github.ibm.mapepire; + +public class BlobRequestData { + + private int replacementIndex; + private int length; + private int offset; + + public BlobRequestData(int replacementIndex, int length, int offset){ + this.replacementIndex = replacementIndex; + this.length = length; + this.offset = offset; + } + + public int getLength() { + return length; + } + + public int getReplacementIndex() { + return replacementIndex; + } + + public int getOffset() { + return offset; + } +} diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 1bfb02f..391ce18 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -12,10 +12,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class DataStreamProcessor implements Runnable { @@ -87,23 +84,32 @@ private int bytesToInt(byte[] bytes, int offset, int length) { public void run(byte[] payload, int offset, int len) { - int cont_id = bytesToInt(payload, offset, 2); - int length = bytesToInt(payload, offset + 2, 4); - - if (payload.length - 6 != length) { - throw new RuntimeException("Invalid binary data recieved."); + int cont_id_length = 2; + int cont_id = bytesToInt(payload, offset, cont_id_length); + + List blobRequestDataArray = new ArrayList<>(); + int curOffset = offset + cont_id_length; + int replacementIndexLength = 1; + int sizeOfInt = 4; + while (curOffset < offset + len){ + int replacementIndex = bytesToInt(payload, curOffset, replacementIndexLength); + curOffset += replacementIndexLength; + + int length = bytesToInt(payload, curOffset, sizeOfInt); + curOffset += sizeOfInt; + + BlobRequestData blobRequestData = new BlobRequestData(replacementIndex, length, curOffset); + blobRequestDataArray.add(blobRequestData); + curOffset += length; } - PrepareSql prev = m_prepStmtMap.get(String.valueOf(cont_id)); if (null == prev) { dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); return; } -// byte[] blob = Arrays.copyOfRange(payload, 6, payload.length); - int blobOffset = 6; try { - RunBlob runBlob = new RunBlob(this, payload, blobOffset, length, prev); + RunBlob runBlob = new RunBlob(payload, blobRequestDataArray, prev); } catch (Exception e) { System.out.println("Caught exception " + e); } diff --git a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java index 036a74d..dc5c359 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java @@ -19,11 +19,21 @@ public class PrepareSql extends BlockRetrievableRequest { private PreparedStatement m_stmt = null; private final PreparedExecute m_executeTask; + private int blobsNeeded = 0; public PrepareSql(final DataStreamProcessor _io, final SystemConnection m_conn, final JsonObject _reqObj, final boolean _isImmediateExecute) { super(_io, m_conn, _reqObj); m_executeTask = _isImmediateExecute ? new PreparedExecute(_io, _reqObj, this) : null; + this.blobsNeeded = getRequestFieldInt("blobsNeeded", 0); + } + + public int getBlobsNeeded() { + return blobsNeeded; + } + + public void setBlobsNeeded(int blobsNeeded) { + this.blobsNeeded = blobsNeeded; } @Override diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 1e2bc9d..89f6e3d 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -1,11 +1,14 @@ package com.github.ibm.mapepire.requests; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.sql.*; import java.util.Arrays; import java.util.LinkedList; +import java.util.List; +import com.github.ibm.mapepire.BlobRequestData; import com.github.ibm.mapepire.DataStreamProcessor; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -17,80 +20,16 @@ public class RunBlob{ private final PrepareSql m_prev; - public RunBlob(final DataStreamProcessor _io, final byte[] binary, final int offset, final int length, final PrepareSql _prev) throws SQLException { + public RunBlob(final byte[] binary, final List blobRequestDataList, final PrepareSql _prev) throws SQLException { m_prev = _prev; PreparedStatement stmt = m_prev.getStatement(); - try (InputStream is = new ByteArrayInputStream(binary, offset, length)) { - stmt.setBinaryStream(1, is, length); - int affectedRows = stmt.executeUpdate(); + for (BlobRequestData blobRequestData: blobRequestDataList){ + int offset = blobRequestData.getOffset(); + int length = blobRequestData.getLength(); + int replacementIndex = blobRequestData.getReplacementIndex(); + InputStream is = new ByteArrayInputStream(binary, offset, length); + stmt.setBinaryStream(replacementIndex, is, length); } - catch (Exception e){ - System.out.println("Caught error " + e); - } - + stmt.execute(); } - -// @Override -// protected void go() throws Exception { -// boolean isBatch = !parms.isEmpty() && parms.get(0).isJsonArray(); -// boolean hasResultSet = false; -// long batchUpdateCount = 0; -// -// PreparedStatement stmt = m_prev.getStatement(); -// if (isBatch) { -// JsonArray arr = parms.getAsJsonArray(); -// int batch_ops_added = 0; -// for (int i = 0; i < arr.size(); i++) { -// addJsonArrayParameters(stmt, arr.get(i).getAsJsonArray()); -// m_prev.getStatement().addBatch(); -// } -// batch_ops_added += arr.size(); -// addReplyData("batch_added", batch_ops_added); -// long updateCount[] = stmt.executeLargeBatch(); -// batchUpdateCount = Arrays.stream(updateCount).sum(); -// } else{ -// if (parms != null) { -// addJsonArrayParameters(stmt, parms.getAsJsonArray()); -// } -// hasResultSet = stmt.execute(); -// } -// -// if (hasResultSet) { -// this.m_rs = stmt.getResultSet(); -// final int numRows = super.getRequestFieldInt("rows", 1000); -// addReplyData("has_results", true); -// addReplyData("update_count", stmt.getLargeUpdateCount()); -// addReplyData("metadata", getResultMetaDataForResponse()); -// addReplyData("data", getNextDataBlock(numRows)); -// addReplyData("output_parms", getOutputParms(stmt)); -// addReplyData("is_done", isDone()); -// } else { -// addReplyData("data", new LinkedList()); -// addReplyData("has_results", false); -// addReplyData("update_count", batchUpdateCount != 0 ? batchUpdateCount : stmt.getLargeUpdateCount()); -// addReplyData("output_parms", getOutputParms(stmt)); -// addReplyData("is_done", m_isDone = true); -// } -// } - - private void addJsonArrayParameters(PreparedStatement stmt, JsonArray arr) throws SQLException { - for (int i = 1; i <= arr.size(); ++i) { - JsonElement element = arr.get(-1 + i); - if (element.isJsonNull()) { - stmt.setNull(i, Types.NULL); - } else { - if (stmt instanceof CallableStatement - && ParameterMetaData.parameterModeOut == stmt.getParameterMetaData().getParameterMode(i)) { - ((CallableStatement) stmt).registerOutParameter(i, stmt.getParameterMetaData().getParameterType(i)); - } else if (stmt instanceof CallableStatement - && ParameterMetaData.parameterModeInOut == stmt.getParameterMetaData().getParameterMode(i)) { - ((CallableStatement) stmt).registerOutParameter(i, stmt.getParameterMetaData().getParameterType(i)); - stmt.setString(i, element.getAsString()); - } else { - stmt.setString(i, element.getAsString()); - } - } - } - } - } From 57921ce3658f9c0d56c983bcee4b713dfbd3305c Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Fri, 22 Aug 2025 14:50:46 -0400 Subject: [PATCH 45/57] add other params as well --- .../github/ibm/mapepire/requests/PrepareSql.java | 14 +++++++++++--- .../com/github/ibm/mapepire/requests/RunBlob.java | 5 +++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java index dc5c359..8359cfd 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java @@ -32,6 +32,10 @@ public int getBlobsNeeded() { return blobsNeeded; } + public PreparedExecute getM_executeTask(){ + return m_executeTask; + } + public void setBlobsNeeded(int blobsNeeded) { this.blobsNeeded = blobsNeeded; } @@ -82,12 +86,16 @@ public void go() throws Exception { addReplyData("metadata", metaData); if (null != m_executeTask) { - m_executeTask.go(); - this.m_rs = m_executeTask.m_rs; - mergeReplyData(m_executeTask); + executeTask(); } } + public void executeTask() throws Exception { + m_executeTask.go(); + this.m_rs = m_executeTask.m_rs; + mergeReplyData(m_executeTask); + } + private static String getDb2ParameterName(PreparedStatement _stmt, int _i) throws SQLException { if (_stmt instanceof AS400JDBCPreparedStatement) { return ((AS400JDBCPreparedStatement) _stmt).getDB2ParameterName(_i); diff --git a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java index 89f6e3d..bcf98d5 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java +++ b/src/main/java/com/github/ibm/mapepire/requests/RunBlob.java @@ -20,7 +20,7 @@ public class RunBlob{ private final PrepareSql m_prev; - public RunBlob(final byte[] binary, final List blobRequestDataList, final PrepareSql _prev) throws SQLException { + public RunBlob(final byte[] binary, final List blobRequestDataList, final PrepareSql _prev) throws Exception { m_prev = _prev; PreparedStatement stmt = m_prev.getStatement(); for (BlobRequestData blobRequestData: blobRequestDataList){ @@ -29,7 +29,8 @@ public RunBlob(final byte[] binary, final List blobRequestDataL int replacementIndex = blobRequestData.getReplacementIndex(); InputStream is = new ByteArrayInputStream(binary, offset, length); stmt.setBinaryStream(replacementIndex, is, length); + } - stmt.execute(); + _prev.executeTask(); } } From 5842b4cc7b32ada4f6b76311b32a4dd11106bd1c Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Fri, 22 Aug 2025 14:58:34 -0400 Subject: [PATCH 46/57] Change preparesql logic --- .../java/com/github/ibm/mapepire/requests/PrepareSql.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java index 8359cfd..4193898 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java +++ b/src/main/java/com/github/ibm/mapepire/requests/PrepareSql.java @@ -20,11 +20,13 @@ public class PrepareSql extends BlockRetrievableRequest { private PreparedStatement m_stmt = null; private final PreparedExecute m_executeTask; private int blobsNeeded = 0; + private final boolean isImmediateExecute; public PrepareSql(final DataStreamProcessor _io, final SystemConnection m_conn, final JsonObject _reqObj, final boolean _isImmediateExecute) { super(_io, m_conn, _reqObj); - m_executeTask = _isImmediateExecute ? new PreparedExecute(_io, _reqObj, this) : null; + this.isImmediateExecute = _isImmediateExecute; + m_executeTask = new PreparedExecute(_io, _reqObj, this); this.blobsNeeded = getRequestFieldInt("blobsNeeded", 0); } @@ -85,7 +87,7 @@ public void go() throws Exception { } addReplyData("metadata", metaData); - if (null != m_executeTask) { + if (isImmediateExecute) { executeTask(); } } @@ -121,7 +123,7 @@ PreparedStatement getStatement() { } @Override public boolean isDone() { - if(null != m_executeTask) { + if (null != m_executeTask) { return m_isDone || m_executeTask.isDone(); } // This means that the request was _only_ for a prepare, so yes we're done if we haven't fetched any data From 22978b570c0d1c0ee1540747515673d8d9989bbb Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 11:03:17 -0400 Subject: [PATCH 47/57] Move blobs needed count out of rows --- .../github/ibm/mapepire/requests/BlockRetrievableRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 4e9a983..0e73be8 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -110,6 +110,7 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ return ret.setDone(true); } final List blobResponseDataArray = new ArrayList<>(); + int blobsNeeded = 0; for (int i = 0; i < _numRows; ++i) { if (!_rs.next()) { ret.setDone(true); @@ -124,7 +125,6 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ final LinkedHashMap mapRowData = new LinkedHashMap(); final LinkedList terseRowData = new LinkedList(); final int numCols = _rs.getMetaData().getColumnCount(); - int blobsNeeded = 0; int rowId = i; mapRowData.put("rowId", rowId); for (int col = 1; col <= numCols; ++col) { From 2f71e1e2f0e64688771850e8f3a6c4d30cbc1816 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 11:08:19 -0400 Subject: [PATCH 48/57] send response data in seperate thread --- .../ibm/mapepire/requests/BlockRetrievableRequest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 0e73be8..f2eea54 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -146,7 +146,14 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ int blobLength = (int) ((Blob) cellData).length(); // InputStream is = _rs.getBinaryStream(col); BlobResponseData blobResponseData = new BlobResponseData((Blob)cellData, column, rowId, blobLength); - m_io.sendResponse(this.getId(), blobResponseData); + new Thread(() -> { + try { + m_io.sendResponse(this.getId(), blobResponseData); + } catch (Exception e) { + e.printStackTrace(); // or log it properly + } + }).start(); +// m_io.sendResponse(this.getId(), blobResponseData); // blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } From cf18f9a803a231b427be798a8d1ac65bae3e37d9 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 11:16:13 -0400 Subject: [PATCH 49/57] put binary sender in seperate thread --- .../mapepire/requests/BlockRetrievableRequest.java | 10 ++-------- .../com/github/ibm/mapepire/ws/DbWebsocketClient.java | 11 ++++++++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index f2eea54..899720f 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -146,14 +146,8 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ int blobLength = (int) ((Blob) cellData).length(); // InputStream is = _rs.getBinaryStream(col); BlobResponseData blobResponseData = new BlobResponseData((Blob)cellData, column, rowId, blobLength); - new Thread(() -> { - try { - m_io.sendResponse(this.getId(), blobResponseData); - } catch (Exception e) { - e.printStackTrace(); // or log it properly - } - }).start(); -// m_io.sendResponse(this.getId(), blobResponseData); + + m_io.sendResponse(this.getId(), blobResponseData); // blobResponseDataArray.add(blobResponseData); // cellDataForResponse = _rs.getBytes(col); } diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 1473c68..aa5bb2a 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -69,7 +69,16 @@ public void awaitClosure() throws InterruptedException { } private DataStreamProcessor getDataStreamProcessor(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { - BinarySender binarySender = (data, isLast) -> getRemote().sendPartialBytes(data, isLast); +// BinarySender binarySender = (data, isLast) -> getRemote().sendPartialBytes(data, isLast); + BinarySender binarySender = (data, isLast) -> { + new Thread(() -> { + try { + getRemote().sendPartialBytes(data, isLast); + } catch (IOException e) { + e.printStackTrace(); + } + }).start(); + }; InputStream in = new ByteArrayInputStream(new byte[0]); OutputStream outStreamText = new OutputStream() { From f70a0f943c1c2280e314a8571e8bb57097b4bb48 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 11:45:28 -0400 Subject: [PATCH 50/57] use async sender --- .../github/ibm/mapepire/DataStreamProcessor.java | 5 +++-- .../github/ibm/mapepire/ws/DbWebsocketClient.java | 13 +++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 391ce18..d29c699 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -1,6 +1,7 @@ package com.github.ibm.mapepire; import com.github.ibm.mapepire.requests.*; +import com.github.ibm.mapepire.ws.AsyncSender; import com.github.ibm.mapepire.ws.BinarySender; import com.github.ibm.mapepire.ws.DbWebsocketClient; import com.github.theprez.jcmdutils.StringUtils; @@ -25,10 +26,10 @@ public class DataStreamProcessor implements Runnable { private final Map m_queriesMap = new HashMap(); private final Map m_prepStmtMap = new HashMap(); private final boolean m_isTestMode; - private final BinarySender m_binarySender; + private final AsyncSender m_binarySender; // private final DbWebsocketClient.BinarySender m_binarySender; - public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final BinarySender binarySender, final SystemConnection _conn, + public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final AsyncSender binarySender, final SystemConnection _conn, boolean _isTestMode) throws UnsupportedEncodingException { m_in = new BufferedReader(new InputStreamReader(_in, "UTF-8")); diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index aa5bb2a..49b9388 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -70,15 +70,8 @@ public void awaitClosure() throws InterruptedException { private DataStreamProcessor getDataStreamProcessor(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { // BinarySender binarySender = (data, isLast) -> getRemote().sendPartialBytes(data, isLast); - BinarySender binarySender = (data, isLast) -> { - new Thread(() -> { - try { - getRemote().sendPartialBytes(data, isLast); - } catch (IOException e) { - e.printStackTrace(); - } - }).start(); - }; + AsyncSender asyncSender = new AsyncSender(this); + InputStream in = new ByteArrayInputStream(new byte[0]); OutputStream outStreamText = new OutputStream() { @@ -111,6 +104,6 @@ public synchronized void flush() throws IOException { PrintStream out = new PrintStream(outStreamText); - return new DataStreamProcessor(in, out, binarySender, conn, false); + return new DataStreamProcessor(in, out, asyncSender, conn, false); } } From 3b9fb26f91421e9d6cea834022ecf5ce6f343887 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 11:46:39 -0400 Subject: [PATCH 51/57] use async sender --- .../github/ibm/mapepire/ws/AsyncSender.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java diff --git a/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java b/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java new file mode 100644 index 0000000..9865811 --- /dev/null +++ b/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java @@ -0,0 +1,33 @@ +package com.github.ibm.mapepire.ws; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.RemoteEndpoint; +import org.eclipse.jetty.websocket.api.WebSocketAdapter; + +public class AsyncSender { + private final ExecutorService executor = Executors.newFixedThreadPool(1); + private final WebSocketAdapter session; + + public AsyncSender(WebSocketAdapter session) { + this.session = session; + } + + public void send(ByteBuffer buffer, boolean isLast) { + executor.submit(() -> { + try { + RemoteEndpoint remote = session.getRemote(); + remote.sendPartialBytes(buffer, isLast); + } catch (Exception e) { + e.printStackTrace(); // or better logging + } + }); + } + + public void shutdown() { + executor.shutdown(); + } +} From b3b60d50aac839c22875682e99cbc1dc2557b72c Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 12:07:38 -0400 Subject: [PATCH 52/57] wait until async send complete --- .../ibm/mapepire/DataStreamProcessor.java | 3 ++- .../github/ibm/mapepire/ws/AsyncSender.java | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index d29c699..7d163d3 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -14,6 +14,7 @@ import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.*; +import java.util.concurrent.ExecutionException; public class DataStreamProcessor implements Runnable { @@ -352,7 +353,7 @@ public void sendResponse(final String id, final BlobResponseData blobResponseDat } } - private void sendByteBuffer(byte[] buffer, int bytesRead, boolean isFinal) throws IOException { + private void sendByteBuffer(byte[] buffer, int bytesRead, boolean isFinal) { // Wrap only the bytes actually read ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); diff --git a/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java b/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java index 9865811..3805326 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java +++ b/src/main/java/com/github/ibm/mapepire/ws/AsyncSender.java @@ -1,6 +1,8 @@ package com.github.ibm.mapepire.ws; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -17,14 +19,20 @@ public AsyncSender(WebSocketAdapter session) { } public void send(ByteBuffer buffer, boolean isLast) { - executor.submit(() -> { - try { + try { + + executor.submit(() -> { RemoteEndpoint remote = session.getRemote(); - remote.sendPartialBytes(buffer, isLast); - } catch (Exception e) { - e.printStackTrace(); // or better logging + try { + remote.sendPartialBytes(buffer, isLast); + } catch (IOException e) { + throw new RuntimeException(e); + } } - }); + ).get();} + catch (Exception e) { + e.printStackTrace(); // or better logging + } } public void shutdown() { From ba7882b06228d908ae0f9bb9259eb6e909ec18ea Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 12:25:09 -0400 Subject: [PATCH 53/57] Make byte buffer larger --- src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 7d163d3..3729d19 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -286,7 +286,7 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep public void sendResponse(final String id, final BlobResponseData blobResponseData) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; - byte[] buffer = new byte[4 * 1024 * 1024]; + byte[] buffer = new byte[8 * 1024 * 1024]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); // buffer = new byte[8192]; From f8dcda65ba341a723eb742ef61e2115aab6312b1 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 12:28:13 -0400 Subject: [PATCH 54/57] change back to 4mb --- src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 3729d19..7d163d3 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -286,7 +286,7 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep public void sendResponse(final String id, final BlobResponseData blobResponseData) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; - byte[] buffer = new byte[8 * 1024 * 1024]; + byte[] buffer = new byte[4 * 1024 * 1024]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); // buffer = new byte[8192]; From 27145e2b899d4b2559b3cdf0b342399f37f41eee Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Mon, 25 Aug 2025 12:32:13 -0400 Subject: [PATCH 55/57] fix tests --- src/test/java/BlobTest.java | 61 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/test/java/BlobTest.java b/src/test/java/BlobTest.java index 6c13ff0..399482a 100644 --- a/src/test/java/BlobTest.java +++ b/src/test/java/BlobTest.java @@ -1,6 +1,7 @@ import com.github.ibm.mapepire.DataStreamProcessor; import com.github.ibm.mapepire.requests.BlockRetrievableRequest; -import com.github.ibm.mapepire.ws.BinarySender; +import com.github.ibm.mapepire.ws.AsyncSender; +//import com.github.ibm.mapepire.ws.asyncSender; import com.google.gson.JsonObject; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -124,8 +125,8 @@ void testBlobColumnInResultSet() throws Exception { public void testSendingSingleRowWithBlob() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -143,13 +144,13 @@ public void testSendingSingleRowWithBlob() throws SQLException, IOException { MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 1, false); - verify(binarySender, times(1)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(1)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); // Verify first call ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -163,8 +164,8 @@ public void testSendingSingleRowWithBlob() throws SQLException, IOException { public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -192,13 +193,13 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 2, false); - verify(binarySender, times(2)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(2)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); // Verify first call ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -220,8 +221,8 @@ public void testSendingMultipleRowsWithBlobs() throws SQLException, IOException public void testSingleRowMultipleBlobs() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "123"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -258,13 +259,13 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 1, false); - verify(binarySender, times(2)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(2)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(2)).send(bufferCaptor.capture(), booleanCaptor.capture()); // Verify first call ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -285,8 +286,8 @@ public void testSingleRowMultipleBlobs() throws SQLException, IOException { public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "query7"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -344,13 +345,13 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 2, false); - verify(binarySender, times(4)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(4)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(4)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(4)).send(bufferCaptor.capture(), booleanCaptor.capture()); // Verify first call ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -386,8 +387,8 @@ public void testMultipleRowsMultipleBlobsPerRow() throws SQLException, IOExcepti public void testSendingSingleRowWithEmptyBlob() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -404,13 +405,13 @@ public void testSendingSingleRowWithEmptyBlob() throws SQLException, IOException MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 1, false); - verify(binarySender, times(0)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(0)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(0)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(0)).send(bufferCaptor.capture(), booleanCaptor.capture()); } @@ -418,8 +419,8 @@ public void testSendingSingleRowWithEmptyBlob() throws SQLException, IOException public void testSendingSingleRowWithBlobandNonblobColumns() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -441,13 +442,13 @@ public void testSendingSingleRowWithBlobandNonblobColumns() throws SQLException, MockResultSet rs = new MockResultSet(rows, rowsByIndex); block.getNextDataBlockUsage(rs, 1, false); - verify(binarySender, times(1)).send(any(ByteBuffer.class), anyBoolean()); + verify(asyncSender, times(1)).send(any(ByteBuffer.class), anyBoolean()); // Assert - capture arguments ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); - verify(binarySender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); + verify(asyncSender, times(1)).send(bufferCaptor.capture(), booleanCaptor.capture()); // Verify first call ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -462,8 +463,8 @@ public void testSendingSingleRowWithBlobandNonblobColumns() throws SQLException, public void testSendingSingleRowWithLongBlob() throws SQLException, IOException { JsonObject obj = new JsonObject(); obj.addProperty("id", "12345"); - BinarySender binarySender = mock(BinarySender.class); - final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, binarySender, null, true); + AsyncSender asyncSender = mock(AsyncSender.class); + final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, asyncSender, null, true); BlockRetrievableRequestImpl block = new BlockRetrievableRequestImpl(io, null, obj); @@ -498,13 +499,13 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException List> rows = Arrays.asList(row1); MockResultSet rs = new MockResultSet(rows, rowsByIndex); // -// verify(binarySender, times(641)).send(any(ByteBuffer.class), anyBoolean()); +// verify(asyncSender, times(641)).send(any(ByteBuffer.class), anyBoolean()); // // // Assert - capture arguments // ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class); // ArgumentCaptor booleanCaptor = ArgumentCaptor.forClass(Boolean.class); // -// verify(binarySender, times(641)).send(bufferCaptor.capture(), booleanCaptor.capture()); +// verify(asyncSender, times(641)).send(bufferCaptor.capture(), booleanCaptor.capture()); // // // Verify first call // ByteBuffer firstBuf = bufferCaptor.getAllValues().get(0); @@ -526,7 +527,7 @@ public void testSendingSingleRowWithLongBlob() throws SQLException, IOException capturedFlags.add(isFinal); return null; - }).when(binarySender).send(any(ByteBuffer.class), anyBoolean()); + }).when(asyncSender).send(any(ByteBuffer.class), anyBoolean()); block.getNextDataBlockUsage(rs, 1, false); byte[] actualFirst21 = Arrays.copyOfRange(capturedBuffers.get(0), 0, 25); From 37f8543cc161fd4870fce9cd6e74504dc65c57cc Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 26 Aug 2025 13:38:39 -0400 Subject: [PATCH 56/57] cleanup --- .../github/ibm/mapepire/ClientRequest.java | 2 +- .../ibm/mapepire/DataStreamProcessor.java | 153 ++++++------------ .../github/ibm/mapepire/MapepireServer.java | 2 - .../java/com/github/ibm/mapepire/Version.java | 2 +- .../requests/BlockRetrievableRequest.java | 5 - .../ibm/mapepire/ws/DbWebsocketClient.java | 14 -- 6 files changed, 51 insertions(+), 127 deletions(-) diff --git a/src/main/java/com/github/ibm/mapepire/ClientRequest.java b/src/main/java/com/github/ibm/mapepire/ClientRequest.java index a25d9fb..9b1b7b9 100644 --- a/src/main/java/com/github/ibm/mapepire/ClientRequest.java +++ b/src/main/java/com/github/ibm/mapepire/ClientRequest.java @@ -122,7 +122,7 @@ private static String getErrorStringFromException(Throwable _e) { return "Internal Error: " + _e.getClass().getSimpleName(); } - protected void sendreply() throws UnsupportedEncodingException, IOException { + protected void sendreply() throws IOException { final Gson l = new GsonBuilder().serializeNulls().create(); final String json = l.toJson(replyData); replyData.clear(); diff --git a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java index 7d163d3..45604b9 100644 --- a/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java +++ b/src/main/java/com/github/ibm/mapepire/DataStreamProcessor.java @@ -2,8 +2,6 @@ import com.github.ibm.mapepire.requests.*; import com.github.ibm.mapepire.ws.AsyncSender; -import com.github.ibm.mapepire.ws.BinarySender; -import com.github.ibm.mapepire.ws.DbWebsocketClient; import com.github.theprez.jcmdutils.StringUtils; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -14,7 +12,6 @@ import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.*; -import java.util.concurrent.ExecutionException; public class DataStreamProcessor implements Runnable { @@ -28,7 +25,6 @@ public class DataStreamProcessor implements Runnable { private final Map m_prepStmtMap = new HashMap(); private final boolean m_isTestMode; private final AsyncSender m_binarySender; -// private final DbWebsocketClient.BinarySender m_binarySender; public DataStreamProcessor(final InputStream _in, final PrintStream _outText, final AsyncSender binarySender, final SystemConnection _conn, boolean _isTestMode) @@ -93,7 +89,7 @@ public void run(byte[] payload, int offset, int len) { int curOffset = offset + cont_id_length; int replacementIndexLength = 1; int sizeOfInt = 4; - while (curOffset < offset + len){ + while (curOffset < offset + len) { int replacementIndex = bytesToInt(payload, curOffset, replacementIndexLength); curOffset += replacementIndexLength; @@ -117,30 +113,6 @@ public void run(byte[] payload, int offset, int len) { } } - -// public void run(byte[] binary) { -// int cont_id = bytesToInt(binary, 0, 2); -// int length = bytesToInt(binary, 2, 4); -// -// if (binary.length - 6 != length) { -// throw new RuntimeException("Invalid binary data recieved."); -// } -// -// -// PrepareSql prev = m_prepStmtMap.get(String.valueOf(cont_id)); -// if (null == prev) { -// dispatch(new BadReq(this, m_conn, null, "invalid correlation ID")); -// return; -// } -// byte[] blob = Arrays.copyOfRange(binary, 6, binary.length); -// try { -// RunBlob runBlob = new RunBlob(this, blob, prev); -// } catch (Exception e) { -// System.out.println("Caught exception " + e); -// } -// -// } - public void run(String requestString) { final JsonElement reqElement; final JsonObject reqObj; @@ -210,12 +182,6 @@ public void run(String requestString) { m_prepStmtMap.remove(cont_id.getAsString()); break; } -// case "blob": { -// final RunBlob blob = new RunBlob(this, m_conn, reqObj); -// m_queriesMap.put(runSqlReq.getId(), runSqlReq); -// dispatch(runSqlReq); -// break; -// } case "execute": if (null == cont_id) { dispatch(new BadReq(this, m_conn, reqObj, "Correlation ID not specified")); @@ -266,97 +232,76 @@ public void sendResponse(final String _response) throws UnsupportedEncodingExcep } } -// public void sendResponse(final InputStream is, final String id) throws UnsupportedEncodingException, IOException { -// synchronized (s_replyWriterLock) { -// byte[] buffer = new byte[8192]; -// int bytesRead; -// boolean isFinal; -// while ((bytesRead = is.read(buffer)) != -1) { -// // Wrap only the bytes actually read -// ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); -// -// // Check if this is the last chunk -// isFinal = is.available() == 0; -// -// m_binarySender.send(byteBuffer, isFinal); -// } -// } -// } - public void sendResponse(final String id, final BlobResponseData blobResponseData) throws IOException, SQLException { synchronized (s_replyWriterLock) { int curOffset = 0; byte[] buffer = new byte[4 * 1024 * 1024]; byte[] idBytes = id.getBytes(StandardCharsets.UTF_8); -// buffer = new byte[8192]; - curOffset = 0; - String columnName = blobResponseData.getColumnName(); - InputStream is = blobResponseData.getBlob().getBinaryStream(); - int rowId = blobResponseData.getRowId(); + curOffset = 0; + String columnName = blobResponseData.getColumnName(); + InputStream is = blobResponseData.getBlob().getBinaryStream(); + int rowId = blobResponseData.getRowId(); - byte[] columnNameBytes = columnName.getBytes(StandardCharsets.UTF_8); + byte[] columnNameBytes = columnName.getBytes(StandardCharsets.UTF_8); - if (idBytes.length > 255) { - throw new IllegalArgumentException("ID too long to encode in one byte length"); - } + if (idBytes.length > 255) { + throw new IllegalArgumentException("ID too long to encode in one byte length"); + } - // First byte is the length of the ID - buffer[curOffset] = (byte) idBytes.length; - curOffset += 1; - // Copy ID bytes after the length byte - System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); - curOffset += idBytes.length; - - - // Copy rowId - ByteBuffer rowIdBuffer = ByteBuffer.allocate(4); - rowIdBuffer.putInt(rowId); // default is big-endian - byte[] rowIdBytes = rowIdBuffer.array(); - for (int j = 0; j < 4; j++){ - buffer[curOffset] = rowIdBytes[j]; - curOffset += 1; - } + // First byte is the length of the ID + buffer[curOffset] = (byte) idBytes.length; + curOffset += 1; - // Copy column length - buffer[curOffset] = (byte) columnName.length(); + // Copy ID bytes after the length byte + System.arraycopy(idBytes, 0, buffer, curOffset, idBytes.length); + curOffset += idBytes.length; + + // Copy rowId + ByteBuffer rowIdBuffer = ByteBuffer.allocate(4); + rowIdBuffer.putInt(rowId); // default is big-endian + byte[] rowIdBytes = rowIdBuffer.array(); + for (int j = 0; j < 4; j++) { + buffer[curOffset] = rowIdBytes[j]; curOffset += 1; + } - // copy column name - System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); - curOffset += columnNameBytes.length; - - // Copy blob length - ByteBuffer blobLength = ByteBuffer.allocate(4); - blobLength.putInt(blobResponseData.getLength()); // default is big-endian - byte[] bytes = blobLength.array(); - for (int j = 0; j < 4; j++){ - buffer[curOffset] = bytes[j]; - curOffset += 1; - } + // Copy column length + buffer[curOffset] = (byte) columnName.length(); + curOffset += 1; - int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); - curOffset += bytesRead; - int totalBytesRead = bytesRead; - if (bytesRead != -1) { - boolean isFinal = totalBytesRead == blobResponseData.getLength(); - sendByteBuffer(buffer, curOffset, isFinal); - } + // copy column name + System.arraycopy(columnNameBytes, 0, buffer, curOffset, columnNameBytes.length); + curOffset += columnNameBytes.length; - while ((bytesRead = is.read(buffer)) != -1) { - totalBytesRead += bytesRead; - boolean isFinal = totalBytesRead == blobResponseData.getLength(); - sendByteBuffer(buffer, bytesRead, isFinal); - } + // Copy blob length + ByteBuffer blobLength = ByteBuffer.allocate(4); + blobLength.putInt(blobResponseData.getLength()); // default is big-endian + byte[] bytes = blobLength.array(); + for (int j = 0; j < 4; j++) { + buffer[curOffset] = bytes[j]; + curOffset += 1; + } + int bytesRead = is.read(buffer, curOffset, buffer.length - curOffset); + curOffset += bytesRead; + int totalBytesRead = bytesRead; + if (bytesRead != -1) { + boolean isFinal = totalBytesRead == blobResponseData.getLength(); + sendByteBuffer(buffer, curOffset, isFinal); + } + while ((bytesRead = is.read(buffer)) != -1) { + totalBytesRead += bytesRead; + boolean isFinal = totalBytesRead == blobResponseData.getLength(); + sendByteBuffer(buffer, bytesRead, isFinal); + } } } private void sendByteBuffer(byte[] buffer, int bytesRead, boolean isFinal) { // Wrap only the bytes actually read ByteBuffer byteBuffer = ByteBuffer.wrap(buffer, 0, bytesRead); - m_binarySender.send(byteBuffer, isFinal); } diff --git a/src/main/java/com/github/ibm/mapepire/MapepireServer.java b/src/main/java/com/github/ibm/mapepire/MapepireServer.java index 7515f99..1b20c66 100644 --- a/src/main/java/com/github/ibm/mapepire/MapepireServer.java +++ b/src/main/java/com/github/ibm/mapepire/MapepireServer.java @@ -68,9 +68,7 @@ public static void main(final String[] _args) { if (testMode) { System.setIn(new FileInputStream(testFile)); } -// DbWebsocketClient.BinarySender binarySender = (data, isLast) -> remote.sendPartialBytes(data, isLast); // final DataStreamProcessor io = new DataStreamProcessor(System.in, System.out, conn, testMode); - // io.run(); } else { s_isSingleMode = false; diff --git a/src/main/java/com/github/ibm/mapepire/Version.java b/src/main/java/com/github/ibm/mapepire/Version.java index 91902c0..c1c6bb0 100644 --- a/src/main/java/com/github/ibm/mapepire/Version.java +++ b/src/main/java/com/github/ibm/mapepire/Version.java @@ -1,5 +1,5 @@ package com.github.ibm.mapepire; public class Version { - static public final String s_compileDateTime = "2025-08-12 20:50:37 (GMT)"; + static public final String s_compileDateTime = "2025-08-25 16:06:39 (GMT)"; static public final String s_version = "2.3.3"; } \ No newline at end of file diff --git a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java index 899720f..73cc171 100644 --- a/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java +++ b/src/main/java/com/github/ibm/mapepire/requests/BlockRetrievableRequest.java @@ -144,12 +144,8 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ } else if (cellData instanceof Blob){ blobsNeeded++; int blobLength = (int) ((Blob) cellData).length(); -// InputStream is = _rs.getBinaryStream(col); BlobResponseData blobResponseData = new BlobResponseData((Blob)cellData, column, rowId, blobLength); - m_io.sendResponse(this.getId(), blobResponseData); -// blobResponseDataArray.add(blobResponseData); -// cellDataForResponse = _rs.getBytes(col); } else { cellDataForResponse = _rs.getString(col); @@ -163,7 +159,6 @@ protected DataBlockFetchResult getNextDataBlock(final ResultSet _rs, final int _ ret.add(_isTerseDataFormat ? terseRowData : mapRowData); addReplyData("blobsNeeded", blobsNeeded); } -// m_io.sendResponse(this.getId(), blobResponseDataArray); return ret; } diff --git a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java index 49b9388..9edd6e8 100644 --- a/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java +++ b/src/main/java/com/github/ibm/mapepire/ws/DbWebsocketClient.java @@ -2,25 +2,17 @@ import com.github.ibm.mapepire.DataStreamProcessor; import com.github.ibm.mapepire.SystemConnection; -import org.eclipse.jetty.websocket.api.RemoteEndpoint; import org.eclipse.jetty.websocket.api.Session; import org.eclipse.jetty.websocket.api.WebSocketAdapter; import org.eclipse.jetty.websocket.api.WebSocketException; import java.io.*; -import java.nio.ByteBuffer; import java.util.concurrent.CountDownLatch; -import com.github.ibm.mapepire.ws.BinarySender; public class DbWebsocketClient extends WebSocketAdapter { private final CountDownLatch closureLatch = new CountDownLatch(1); private final DataStreamProcessor io; - /* @FunctionalInterface - public interface BinarySender { - void send(ByteBuffer buffer, boolean isLast) throws IOException; - }*/ - DbWebsocketClient(String clientHost, String clientAddress, String host, String user, String pass) throws IOException { super(); SystemConnection conn = new SystemConnection(clientHost, clientAddress,host, user, pass); @@ -42,12 +34,7 @@ public void onWebSocketText(String message) { @Override public void onWebSocketBinary(byte[] payload, int offset, int len) { - System.out.println(">>> onWebSocketBinary called with len=" + len); - // Access only the relevant portion of the data -// byte[] binary = Arrays.copyOfRange(payload, offset, offset + len); io.run(payload, offset, len); - - // Now use `message` as needed } @Override @@ -69,7 +56,6 @@ public void awaitClosure() throws InterruptedException { } private DataStreamProcessor getDataStreamProcessor(DbWebsocketClient endpoint, SystemConnection conn) throws UnsupportedEncodingException { -// BinarySender binarySender = (data, isLast) -> getRemote().sendPartialBytes(data, isLast); AsyncSender asyncSender = new AsyncSender(this); InputStream in = new ByteArrayInputStream(new byte[0]); From 69e75b136a407bdc341ad74da4c7471458f931a8 Mon Sep 17 00:00:00 2001 From: Jonathan Zak Date: Tue, 26 Aug 2025 13:40:45 -0400 Subject: [PATCH 57/57] cleanup --- src/test/java/MockResultSet.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/MockResultSet.java b/src/test/java/MockResultSet.java index ca68e73..064e571 100644 --- a/src/test/java/MockResultSet.java +++ b/src/test/java/MockResultSet.java @@ -1030,6 +1030,4 @@ public boolean isWrapperFor(Class iface) throws SQLException { return false; } - - // Add more getters as needed... }