Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/emissary/grpc/GrpcConnectionPlace.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package emissary.grpc;

import emissary.config.Configurator;
import emissary.grpc.pool.ConnectionFactory;
import emissary.grpc.channel.ChannelManager;
import emissary.grpc.retry.RetryHandler;

import com.google.common.util.concurrent.ListenableFuture;
Expand All @@ -27,7 +27,7 @@
* <ul>
* <li>{@code GRPC_HOST} - gRPC service hostname or DNS target, <i>required</i></li>
* <li>{@code GRPC_PORT} - gRPC service port, <i>required</i></li>
* <li>See {@link ConnectionFactory} for supported pooling and gRPC channel configuration keys and defaults.</li>
* <li>See {@link ChannelManager} for supported gRPC channel configuration keys and defaults.</li>
* <li>See {@link RetryHandler} for supported retry configuration keys and defaults.</li>
* </ul>
*/
Expand Down
101 changes: 55 additions & 46 deletions src/main/java/emissary/grpc/GrpcRoutingPlace.java
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
package emissary.grpc;

import emissary.config.Configurator;
import emissary.grpc.channel.ChannelManager;
import emissary.grpc.channel.PooledChannelManager;
import emissary.grpc.invoker.GrpcInvoker;
import emissary.grpc.pool.ConnectionFactory;
import emissary.grpc.pool.PoolException;
import emissary.grpc.retry.RetryHandler;
import emissary.place.ServiceProviderPlace;

import com.google.common.util.concurrent.ListenableFuture;
import com.google.protobuf.Message;
import io.grpc.ChannelCredentials;
import io.grpc.InsecureChannelCredentials;
import io.grpc.ManagedChannel;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import io.grpc.stub.AbstractBlockingStub;
import io.grpc.stub.AbstractFutureStub;
import jakarta.annotation.Nullable;
import org.apache.commons.pool2.ObjectPool;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -38,7 +39,9 @@
* identifier for the given host:port</li>
* <li>{@code GRPC_PORT_{Target-ID}} - gRPC service port, where {@code Target-ID} is the unique identifier for the given
* host:port</li>
* <li>See {@link ConnectionFactory} for supported pooling and gRPC channel configuration keys and defaults.</li>
* <li>{@code GRPC_CHANNEL_MANAGER_CLASS_NAME} - fully qualified class name of the {@link ChannelManager} to use,
* default={@link PooledChannelManager emissary.grpc.channel.PooledChannelManager}</li>
* <li>See {@link ChannelManager} for supported gRPC channel configuration keys and defaults.</li>
* <li>See {@link RetryHandler} for supported retry configuration keys and defaults.</li>
* </ul>
*/
Expand All @@ -50,12 +53,9 @@ public abstract class GrpcRoutingPlace extends ServiceProviderPlace implements I

public static final String GRPC_HOST = "GRPC_HOST_";
public static final String GRPC_PORT = "GRPC_PORT_";
public static final String CHANNEL_MANAGER_CLASS_NAME = ChannelManager.GRPC_CHANNEL_PREFIX + "MANAGER_CLASS_NAME";

protected GrpcInvoker grpcInvoker;

protected final Map<String, String> hostnameTable = new HashMap<>();
protected final Map<String, Integer> portNumberTable = new HashMap<>();
protected final Map<String, ObjectPool<ManagedChannel>> channelPoolTable = new HashMap<>();
protected final Map<String, GrpcInvoker> invokerTable = new HashMap<>();

protected GrpcRoutingPlace() throws IOException {
super();
Expand Down Expand Up @@ -98,28 +98,31 @@ protected GrpcRoutingPlace(@Nullable Configurator configs) throws IOException {
}

private void configureGrpc() {
if (configG == null) {
throw new IllegalStateException("gRPC configurations not found for " + this.getPlaceName());
}
Objects.requireNonNull(configG);

hostnameTable.putAll(getHostnameConfigs());
portNumberTable.putAll(getPortNumberConfigs());
Map<String, String> hosts = getHostnameConfigs();
Map<String, Integer> ports = getPortNumberConfigs();

if (!hostnameTable.keySet().equals(portNumberTable.keySet())) {
if (!hosts.keySet().equals(ports.keySet())) {
throw new IllegalArgumentException("gRPC hostname target-IDs do not match gRPC port number target-IDs");
}

if (hostnameTable.isEmpty()) {
Set<String> targetIds = hosts.keySet();
if (targetIds.isEmpty()) {
throw new NullPointerException(String.format(
"Missing required arguments: %s${Target-ID} and %s${Target-ID}", GRPC_HOST, GRPC_PORT));
}

Set<String> targetIds = hostnameTable.keySet();
RetryHandler retryHandler = new RetryHandler(configG, this.getPlaceName(), this::retryOnException);
String channelManagerName = configG.findStringEntry(CHANNEL_MANAGER_CLASS_NAME, PooledChannelManager.class.getName());
ChannelCredentials channelCredentials = getChannelCredentials(configG);

for (String id : targetIds) {
channelPoolTable.put(id, newConnectionPool(id));
ChannelManager channelManager = ChannelManager.ofSubClass(
channelManagerName, hosts.get(id), ports.get(id), configG, channelCredentials);
GrpcInvoker grpcInvoker = new GrpcInvoker(channelManager, retryHandler);
invokerTable.put(id, grpcInvoker);
}

grpcInvoker = new GrpcInvoker(new RetryHandler(configG, this.getPlaceName(), this::retryOnException));
}

protected Map<String, String> getHostnameConfigs() {
Expand All @@ -131,10 +134,22 @@ protected Map<String, Integer> getPortNumberConfigs() {
.collect(Collectors.toMap(Map.Entry::getKey, entry -> Integer.parseInt(entry.getValue())));
}

/**
* Creates a new {@link ChannelCredentials} object with all security and encryption configurations necessary for
* establishing a gRPC connection. Base class returns {@link InsecureChannelCredentials}, which asserts that no client
* identity, authentication, or encryption is to be used. Subclasses should override this method as necessary.
*
* @param configG any necessary configurations for creating the credentials object
* @return gRPC channel credentials
*/
protected ChannelCredentials getChannelCredentials(Configurator configG) {
return InsecureChannelCredentials.create();
}

/**
* Determines if the {@link RetryHandler} should try again when an exception is thrown during gRPC invocation. Default
* behavior attempts retries for any {@link PoolException} or for any Exception with a {@link Status.Code} in
* {@link #RETRY_GRPC_CODES}. Subclasses may override this behavior.
* behavior attempts retries for any Exception with a {@link Status.Code} in {@link #RETRY_GRPC_CODES}. Subclasses may
* override this behavior.
*
* @param t the Exception thrown during gRPC invocation
* @return {@code true} if the {@link RetryHandler} should try again, otherwise {@code false}
Expand All @@ -144,20 +159,12 @@ protected boolean retryOnException(Throwable t) {
StatusRuntimeException e = (StatusRuntimeException) t;
return RETRY_GRPC_CODES.contains(e.getStatus().getCode());
}
return t instanceof PoolException;
}

private ObjectPool<ManagedChannel> newConnectionPool(String id) {
return newConnectionFactory(id).newConnectionPool();
}

private ConnectionFactory newConnectionFactory(String id) {
return new ConnectionFactory(hostnameTable.get(id), portNumberTable.get(id), Objects.requireNonNull(configG));
return false;
}

/**
* Wrapper method for {@link GrpcInvoker#invoke(ObjectPool, Function, BiFunction, Message)} that executes a unary gRPC
* call to a given endpoint.
* Wrapper method for {@link GrpcInvoker#invoke(Function, BiFunction, Message)} that executes a unary gRPC call to a
* given endpoint.
*
* @param targetId the identifier used in the configs for the given gRPC endpoint
* @param stubFactory function that creates the appropriate gRPC stub from a {@link ManagedChannel}
Expand All @@ -170,12 +177,12 @@ private ConnectionFactory newConnectionFactory(String id) {
*/
protected <Q extends Message, R extends Message, S extends AbstractBlockingStub<S>> R invokeGrpc(
String targetId, Function<ManagedChannel, S> stubFactory, BiFunction<S, Q, R> callLogic, Q request) {
return grpcInvoker.invoke(channelPoolLookup(targetId), stubFactory, callLogic, request);
return getInvoker(targetId).invoke(stubFactory, callLogic, request);
}

/**
* Wrapper method for {@link GrpcInvoker#invokeAsync(ObjectPool, Function, BiFunction, Message)} that executes a unary
* gRPC call to a given endpoint and returns a {@link CompletableFuture future}.
* Wrapper method for {@link GrpcInvoker#invokeAsync(Function, BiFunction, Message)} that executes a unary gRPC call to
* a given endpoint and returns a {@link CompletableFuture future}.
*
* @param targetId the identifier used in the configs for the given gRPC endpoint
* @param stubFactory function that creates the appropriate gRPC stub from a {@link ManagedChannel}
Expand All @@ -188,25 +195,27 @@ protected <Q extends Message, R extends Message, S extends AbstractBlockingStub<
*/
protected <Q extends Message, R extends Message, S extends AbstractFutureStub<S>> CompletableFuture<R> invokeGrpcAsync(
String targetId, Function<ManagedChannel, S> stubFactory, BiFunction<S, Q, ListenableFuture<R>> callLogic, Q request) {
return grpcInvoker.invokeAsync(channelPoolLookup(targetId), stubFactory, callLogic, request);
}

private ObjectPool<ManagedChannel> channelPoolLookup(String targetId) {
return tableLookup(channelPoolTable, targetId);
return getInvoker(targetId).invokeAsync(stubFactory, callLogic, request);
}

public String getHostname(String targetId) {
return tableLookup(hostnameTable, targetId);
return getInvoker(targetId).getHost();
}

public int getPortNumber(String targetId) {
return tableLookup(portNumberTable, targetId);
return getInvoker(targetId).getPort();
}

protected <T> T tableLookup(Map<String, T> table, String targetId) {
if (table.containsKey(targetId)) {
return table.get(targetId);
private GrpcInvoker getInvoker(String targetId) {
if (invokerTable.containsKey(targetId)) {
return invokerTable.get(targetId);
}
throw new IllegalArgumentException(String.format("Target-ID %s was never configured", targetId));
}

@Override
public void shutDown() {
super.shutDown();
invokerTable.values().forEach(GrpcInvoker::close);
}
}
165 changes: 165 additions & 0 deletions src/main/java/emissary/grpc/channel/ChannelManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package emissary.grpc.channel;

import emissary.config.Configurator;

import io.grpc.ChannelCredentials;
import io.grpc.Grpc;
import io.grpc.ManagedChannel;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.concurrent.TimeUnit;

/**
* Abstract base class for managing gRPC {@link io.grpc.ManagedChannel} connections.
* <p>
* Configuration Keys:
* <ul>
* <li>{@code GRPC_CHANNEL_KEEP_ALIVE_MILLIS} - Time to wait before sending a ping on idle, default={@code 60000L}</li>
* <li>{@code GRPC_CHANNEL_KEEP_ALIVE_TIMEOUT_MILLIS} - Timeout for receiving ping ACKs, default={@code 30000L}</li>
* <li>{@code GRPC_CHANNEL_KEEP_ALIVE_WITHOUT_CALLS} - Send pings even when no RPCs are active if {@code true},
* default={@code false}</li>
* <li>{@code GRPC_CHANNEL_LOAD_BALANCING_POLICY} - gRPC load balancing policy, default={@code "round_robin"}</li>
* <li>{@code GRPC_CHANNEL_MAX_INBOUND_MESSAGE_BYTE_SIZE} - Max inbound gRPC message size, default={@code 4194304}</li>
* <li>{@code GRPC_CHANNEL_MAX_INBOUND_METADATA_BYTE_SIZE} - Max inbound gRPC metadata size, default={@code 8192}</li>
* </ul>
*/
public abstract class ChannelManager implements AutoCloseable {
public static final String GRPC_CHANNEL_PREFIX = "GRPC_CHANNEL_";
public static final String KEEP_ALIVE_MILLIS = GRPC_CHANNEL_PREFIX + "KEEP_ALIVE_MILLIS";
public static final String KEEP_ALIVE_TIMEOUT_MILLIS = GRPC_CHANNEL_PREFIX + "KEEP_ALIVE_TIMEOUT_MILLIS";
public static final String KEEP_ALIVE_WITHOUT_CALLS = GRPC_CHANNEL_PREFIX + "KEEP_ALIVE_WITHOUT_CALLS";
public static final String LOAD_BALANCING_POLICY = GRPC_CHANNEL_PREFIX + "LOAD_BALANCING_POLICY";
public static final String MAX_INBOUND_MESSAGE_BYTE_SIZE = GRPC_CHANNEL_PREFIX + "MAX_INBOUND_MESSAGE_BYTE_SIZE";
public static final String MAX_INBOUND_METADATA_BYTE_SIZE = GRPC_CHANNEL_PREFIX + "MAX_INBOUND_METADATA_BYTE_SIZE";

protected static final int MAX_PORT_NUMBER = 0xFFFF;

protected final Logger logger;

protected final String host;
protected final int port;
protected final String target;
protected final long keepAliveMillis;
protected final long keepAliveTimeoutMillis;
protected final boolean keepAliveWithoutCalls;
protected final int maxInboundMessageByteSize;
protected final int maxInboundMetadataByteSize;
protected final String loadBalancingPolicy;
protected final ChannelCredentials channelCredentials;

/**
* Constructs a new gRPC connection manager using the provided host, port, and configuration. Initializes gRPC channel
* properties from the given configuration source.
*
* @param host gRPC service hostname or DNS target
* @param port gRPC service port
* @param configG configuration provider for channel parameters
* @param credentials transport-level credentials for the gRPC server
* @see ChannelManager
* @see <a href="https://docs.microsoft.com/en-us/aspnet/core/grpc/performance?view=aspnetcore-5.0">Source</a> for
* default gRPC configurations.
*/
protected ChannelManager(String host, int port, Configurator configG, ChannelCredentials credentials) {
this.logger = LoggerFactory.getLogger(this.getClass().getName());

this.host = host;
this.port = port;
this.target = createTarget();

// How often (in milliseconds) to send pings when the connection is idle
this.keepAliveMillis = configG.findLongEntry(KEEP_ALIVE_MILLIS, 60000L);

// Time to wait (in milliseconds) for a ping ACK before closing the connection
this.keepAliveTimeoutMillis = configG.findLongEntry(KEEP_ALIVE_TIMEOUT_MILLIS, 30000L);

// Whether to send pings when no RPCs are active
// Note: Seme gRPC services have this set to false and will be noisy if not adjusted
this.keepAliveWithoutCalls = configG.findBooleanEntry(KEEP_ALIVE_WITHOUT_CALLS, false);

// Specifies how the client chooses between multiple backend addresses
// e.g. "pick_first" uses the first address only, "round_robin" cycles through all of them for client-side balancing
this.loadBalancingPolicy = configG.findObjectEntry(
LOAD_BALANCING_POLICY, LoadBalancingPolicy::valueOf, LoadBalancingPolicy.ROUND_ROBIN).toString();

// Max size (in bytes) for incoming messages and message metadata from the server
this.maxInboundMessageByteSize = configG.findIntEntry(MAX_INBOUND_MESSAGE_BYTE_SIZE, 4 << 20); // 4 MiB
this.maxInboundMetadataByteSize = configG.findIntEntry(MAX_INBOUND_METADATA_BYTE_SIZE, 8 << 10); // 8 KiB

this.channelCredentials = credentials;
}

private String createTarget() {
if (StringUtils.isEmpty(host)) {
throw new IllegalArgumentException("Missing required gRPC host configuration");
}
if (port <= 0 || port > MAX_PORT_NUMBER) {
throw new IllegalArgumentException(
String.format("Port \"%d\" is outside valid range [1, %d]", port, MAX_PORT_NUMBER));
}
return host + ":" + port;
}

public static ChannelManager ofSubClass(
String className, String host, int port, Configurator configG, ChannelCredentials credentials) {
try {
return ofSubClass(Class.forName(className).asSubclass(ChannelManager.class), host, port, configG, credentials);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException("Cannot find class: " + className, e);
}
}

public static ChannelManager ofSubClass(
Class<? extends ChannelManager> clz, String host, int port, Configurator configG, ChannelCredentials credentials) {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Attached is a patch to replace reflection with a factory design pattern. Any class subclass without the proper method signature will throw a compile error.
0001-replace-reflection-with-a-factory.patch

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've been using this reflection pattern for awhile. Should I make some new tickets to change them to this factory design pattern as well?

@fbruton fbruton Jun 24, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend it. It will improve maintainability.

return ChannelManagerRegistry.ofSubClass(clz, host, port, configG, credentials);
}

/**
* Creates a new {@link ManagedChannel} instance configured with the current factory settings.
*
* @return a new gRPC channel
*/
protected final ManagedChannel create() {
return Grpc.newChannelBuilder(this.target, this.channelCredentials)
.keepAliveTime(this.keepAliveMillis, TimeUnit.MILLISECONDS)
.keepAliveTimeout(this.keepAliveTimeoutMillis, TimeUnit.MILLISECONDS)
.keepAliveWithoutCalls(this.keepAliveWithoutCalls)
.defaultLoadBalancingPolicy(this.loadBalancingPolicy)
.maxInboundMessageSize(this.maxInboundMessageByteSize)
.maxInboundMetadataSize(this.maxInboundMetadataByteSize)
.build();
}

public abstract ManagedChannel acquire();

public abstract void release(ManagedChannel channel);

public abstract void shutdown(ManagedChannel channel);

@Override
public abstract void close();

public String getHost() {
return host;
}

public int getPort() {
return port;
}

public enum LoadBalancingPolicy {
ROUND_ROBIN("round_robin"), PICK_FIRST("pick_first");

private final String policy;

LoadBalancingPolicy(String policy) {
this.policy = policy;
}

@Override
public String toString() {
return policy;
}
}
}
10 changes: 10 additions & 0 deletions src/main/java/emissary/grpc/channel/ChannelManagerFactory.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package emissary.grpc.channel;

import emissary.config.Configurator;

import io.grpc.ChannelCredentials;

@FunctionalInterface
public interface ChannelManagerFactory {
ChannelManager create(String host, int port, Configurator configG, ChannelCredentials credentials);
}
Loading
Loading