From 2cb9734fbdbd0c04cea488cf289d448ff6a5c7af Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Thu, 1 Feb 2018 20:13:15 -0500 Subject: [PATCH 01/11] wip --- grpc-json-bridge/build.gradle | 1 + .../jared2501/grpc/bridge/GrpcBridge.java | 16 ++ .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 29 +++ .../grpc/bridge/GrpcReflectionClient.java | 18 ++ .../grpc/bridge/GrpcReflectionClientImpl.java | 191 ++++++++++++++++++ .../bridge/GrpcBridgeImplIntegrationTest.java | 116 +++++++++++ test-grpc-service/src/grpc/proto/test.proto | 3 + .../src/grpc/proto/transitive.proto | 12 ++ .../grpc/bridge/test/TestServiceImpl.java | 5 + .../grpc/bridge/test/TestServiceImplTest.java | 2 +- versions.props | 1 + 11 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java create mode 100644 grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java create mode 100644 test-grpc-service/src/grpc/proto/transitive.proto diff --git a/grpc-json-bridge/build.gradle b/grpc-json-bridge/build.gradle index 382c873..c0d49c8 100644 --- a/grpc-json-bridge/build.gradle +++ b/grpc-json-bridge/build.gradle @@ -4,6 +4,7 @@ apply from: rootProject.file('gradle/junit5.gradle') dependencies { compile 'io.grpc:grpc-netty' compile 'io.grpc:grpc-services' + compile 'org.slf4j:slf4j-api' testCompile project(':test-grpc-service') testCompile 'org.assertj:assertj-core' diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java new file mode 100644 index 0000000..f59b64d --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java @@ -0,0 +1,16 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.CompletableFuture; + +public interface GrpcBridge { + /** + * Invokes the specified method on the specified service with the given JSON input, returns the JSON output of the + * result. Note that streaming methods will result in an error state for the returned future. + */ + CompletableFuture invoke(String serviceName, String method, byte[] jsonInput); +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java new file mode 100644 index 0000000..c6ead7b --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -0,0 +1,29 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.protobuf.util.JsonFormat; +import java.util.concurrent.CompletableFuture; + +public final class GrpcBridgeImpl implements GrpcBridge { + + private final GrpcReflectionClient.Provider clientProvider; + + public GrpcBridgeImpl(GrpcReflectionClient.Provider clientProvider) { + this.clientProvider = clientProvider; + } + + @Override + public CompletableFuture invoke(String serviceName, String method, byte[] jsonInput) { + return clientProvider.get(serviceName) + .getTypeRegistry() + .thenApply(typeRegistry -> { + JsonFormat.parser().usingTypeRegistry(typeRegistry); + + return null; + }); + } + +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java new file mode 100644 index 0000000..a7c18ed --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java @@ -0,0 +1,18 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.protobuf.util.JsonFormat; +import java.util.concurrent.CompletableFuture; + +public interface GrpcReflectionClient { + + interface Provider { + GrpcReflectionClient get(String serviceName); + } + + CompletableFuture getTypeRegistry(); + +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java new file mode 100644 index 0000000..3b74056 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java @@ -0,0 +1,191 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeTraverser; +import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import io.grpc.reflection.v1alpha.FileDescriptorResponse; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; +import io.grpc.reflection.v1alpha.ServerReflectionRequest; +import io.grpc.reflection.v1alpha.ServerReflectionResponse; +import io.grpc.reflection.v1alpha.ServiceResponse; +import io.grpc.stub.StreamObserver; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GrpcReflectionClientImpl implements GrpcReflectionClient { + + private static final Logger log = LoggerFactory.getLogger(GrpcReflectionClientImpl.class); + + private final String serviceName; + private final ServerReflectionGrpc.ServerReflectionStub reflection; + + private StreamObserver reqStream; + + public GrpcReflectionClientImpl(String serviceName, ServerReflectionGrpc.ServerReflectionStub reflection) { + this.serviceName = serviceName; + this.reflection = reflection; + } + + // TODO: thread safety?? caching?? + @Override + public CompletableFuture getTypeRegistry() { + CompletableFuture future = new CompletableFuture<>(); + + Set outstandingRequests = Sets.newHashSet(); + Map protosByFileName = Maps.newHashMap(); + + reqStream = reflection.serverReflectionInfo( + new StreamObserver() { + @Override + public void onNext(ServerReflectionResponse response) { + if (future.isCancelled()) { + log.info("Future cancelled, not proceeding..."); + return; + } + + outstandingRequests.remove(response.getOriginalRequest()); + + switch (response.getMessageResponseCase()) { + case LIST_SERVICES_RESPONSE: + requestAllFilesForServices(outstandingRequests, response); + case FILE_DESCRIPTOR_RESPONSE: + requestUnseenDependencyProtos( + outstandingRequests, protosByFileName, response.getFileDescriptorResponse()); + default: + log.error("Unexpected response case: {}", response.getMessageResponseCase()); + } + + if (outstandingRequests.isEmpty()) { + Collection compiledProtos = compileProtos(protosByFileName); + JsonFormat.TypeRegistry.Builder typeRegistry = JsonFormat.TypeRegistry.newBuilder(); + for (Descriptors.FileDescriptor compiledProto : compiledProtos) { + typeRegistry.add(compiledProto.getMessageTypes()); + } + future.complete(typeRegistry.build()); + } + } + + @Override + public void onError(Throwable error) { + log.error( + "Throwable encountered when streaming handling reflecting for service {}", + serviceName, error); + } + + @Override + public void onCompleted() { + log.info("Reflection complete, service {} likely shutting down.", serviceName); + } + }); + + // List all services to initial a download + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setListServices("true") + .build()); + + return future; + } + + private void requestAllFilesForServices( + Set outstandingRequests, ServerReflectionResponse response) { + for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { + makeRequest(outstandingRequests, ServerReflectionRequest.newBuilder() + .setFileContainingSymbol(service.getName()) + .build()); + } + } + + private void requestUnseenDependencyProtos( + Set outstandingRequests, + Map protosByFileName, + FileDescriptorResponse response) { + for (ByteString protoBytes : response.getFileDescriptorProtoList()) { + DescriptorProtos.FileDescriptorProto protoDescriptor; + try { + protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); + } catch (InvalidProtocolBufferException e) { + log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); + continue; + } + + protosByFileName.put(protoDescriptor.getName(), protoDescriptor); + + for (String dependencyFileName : protoDescriptor.getDependencyList()) { + if (!protosByFileName.containsKey(dependencyFileName)) { + makeRequest(outstandingRequests, ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); + } + } + } + } + + private Collection compileProtos( + Map protosByFileName) { + // Find all "roots", where a root is a proto file for which another proto file does not depend on it + Map rootsByFileName = Maps.newHashMap(protosByFileName); + for (DescriptorProtos.FileDescriptorProto proto : protosByFileName.values()) { + for (String dependencyFileName : proto.getDependencyList()) { + protosByFileName.remove(dependencyFileName); + } + } + + Map compiledProtosByFileName = Maps.newHashMap(); + + // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto + // file if it has not already been compiled + TreeTraverser treeTraverser = + new TreeTraverser() { + @Override + public Iterable children( + DescriptorProtos.FileDescriptorProto root) { + return root.getDependencyList() + .stream() + // Note: skip visiting dependencies if they have already been compiled + .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) + .map(protosByFileName::get) + .collect(Collectors.toSet()); + } + }; + for (DescriptorProtos.FileDescriptorProto root : rootsByFileName.values()) { + for (DescriptorProtos.FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + Descriptors.FileDescriptor[] dependencies = proto.getDependencyList() + .stream() + .map(compiledProtosByFileName::get) + .toArray(Descriptors.FileDescriptor[]::new); + + Descriptors.FileDescriptor compiledProto; + try { + compiledProto = Descriptors.FileDescriptor.buildFrom(proto, dependencies); + } catch (Descriptors.DescriptorValidationException e) { + log.warn("Exception encountered when building proto... skipping", e); + continue; + } + + compiledProtosByFileName.put(proto.getName(), compiledProto); + } + } + + return compiledProtosByFileName.values(); + } + + private void makeRequest(Set outstandingRequests, ServerReflectionRequest request) { + outstandingRequests.add(request); + reqStream.onNext(request); + } + +} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java new file mode 100644 index 0000000..f1a867a --- /dev/null +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -0,0 +1,116 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.github.jared2501.grpc.bridge.test.TestServiceImpl; +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; +import io.grpc.reflection.v1alpha.ServerReflectionRequest; +import io.grpc.reflection.v1alpha.ServerReflectionResponse; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class GrpcBridgeImplIntegrationTest { + + private Server server; + private ManagedChannel channel; + + @BeforeEach + void setUp() throws IOException { + server = InProcessServerBuilder.forName("test") + .addService(TestServiceImpl.INSTANCE) + .addService(ProtoReflectionService.newInstance()) + .build() + .start(); + channel = InProcessChannelBuilder.forName("test").build(); + } + + @AfterEach + void tearDown() throws InterruptedException { + channel.shutdown(); + channel.awaitTermination(5, TimeUnit.HOURS); + server.shutdown(); + server.awaitTermination(); + } + + StreamObserver reqStream; + + @Test + void barbaz() throws InterruptedException, Descriptors.DescriptorValidationException { + ServerReflectionGrpc.ServerReflectionStub reflection = ServerReflectionGrpc.newStub(channel); + + // Steps: + // 1. List all services + kick of requests for their protos + // 2. For every proto get it's dependencies and if we don't have them then request them and record graph + // 3. Compile all protos + // 4. Build a type registry + // 5. Notify the consumer + + Map protoDescriptors = Maps.newHashMap(); + + reqStream = reflection.serverReflectionInfo( + new StreamObserver() { + @Override + public void onNext(ServerReflectionResponse value) { + System.out.println("all protos: " + protoDescriptors); + + for (ByteString protoBytes : value.getFileDescriptorResponse().getFileDescriptorProtoList()) { + DescriptorProtos.FileDescriptorProto protoDescriptor; + try { + protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + protoDescriptors.put(protoDescriptor.getName(), protoDescriptor); + + for (String dependencyFileName : protoDescriptor.getDependencyList()) { + if (!protoDescriptors.containsKey(dependencyFileName)) { + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); + } + } + } + } + + @Override + public void onError(Throwable t) { + t.printStackTrace(); + } + + @Override + public void onCompleted() { + System.out.println("done!"); + } + }); + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setFileContainingSymbol("com.github.jared2501.grpc.bridge.test.TestService") + .build()); + + Descriptors.FileDescriptor fileDescriptor = Descriptors.FileDescriptor.buildFrom(null, null); + JsonFormat.TypeRegistry.newBuilder() + .add(fileDescriptor.getMessageTypes()) + .build(); + + Thread.sleep(5000); + } + +} diff --git a/test-grpc-service/src/grpc/proto/test.proto b/test-grpc-service/src/grpc/proto/test.proto index 605ce3a..5ee8ffb 100644 --- a/test-grpc-service/src/grpc/proto/test.proto +++ b/test-grpc-service/src/grpc/proto/test.proto @@ -2,11 +2,14 @@ syntax = "proto3"; package com.github.jared2501.grpc.bridge.test; +import "transitive.proto"; + option java_package = "com.github.jared2501.grpc.bridge.test"; option java_multiple_files = true; message TestMessage { string message = 1; + TransitiveEmpty empty = 2; // Tests transitive imports } // A simple test service that will echo a TestMessage. diff --git a/test-grpc-service/src/grpc/proto/transitive.proto b/test-grpc-service/src/grpc/proto/transitive.proto new file mode 100644 index 0000000..1337250 --- /dev/null +++ b/test-grpc-service/src/grpc/proto/transitive.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package com.github.jared2501.grpc.bridge.test; + +import "google/protobuf/empty.proto"; + +option java_package = "com.github.jared2501.grpc.bridge.test"; +option java_multiple_files = true; + +message TransitiveEmpty { + google.protobuf.Empty empty = 1; +} diff --git a/test-grpc-service/src/main/java/com/github/jared2501/grpc/bridge/test/TestServiceImpl.java b/test-grpc-service/src/main/java/com/github/jared2501/grpc/bridge/test/TestServiceImpl.java index 18db38d..dc573d7 100644 --- a/test-grpc-service/src/main/java/com/github/jared2501/grpc/bridge/test/TestServiceImpl.java +++ b/test-grpc-service/src/main/java/com/github/jared2501/grpc/bridge/test/TestServiceImpl.java @@ -7,6 +7,11 @@ import io.grpc.stub.StreamObserver; public final class TestServiceImpl extends TestServiceGrpc.TestServiceImplBase { + + public static final TestServiceImpl INSTANCE = new TestServiceImpl(); + + private TestServiceImpl() {} + @Override public void unaryReqUnaryResp(TestMessage request, StreamObserver responseObserver) { responseObserver.onNext(request); diff --git a/test-grpc-service/src/test/java/com/github/jared2501/grpc/bridge/test/TestServiceImplTest.java b/test-grpc-service/src/test/java/com/github/jared2501/grpc/bridge/test/TestServiceImplTest.java index 4e2e2f2..1fe4272 100644 --- a/test-grpc-service/src/test/java/com/github/jared2501/grpc/bridge/test/TestServiceImplTest.java +++ b/test-grpc-service/src/test/java/com/github/jared2501/grpc/bridge/test/TestServiceImplTest.java @@ -24,7 +24,7 @@ class TestServiceImplTest { @BeforeEach void before() { MockitoAnnotations.initMocks(this); - testService = new TestServiceImpl(); + testService = TestServiceImpl.INSTANCE; } @Test diff --git a/versions.props b/versions.props index f94c40a..edc30fb 100644 --- a/versions.props +++ b/versions.props @@ -4,3 +4,4 @@ org.apiguardian:apiguardian-api = 1.0.0 org.assertj:assertj-core = 3.6.2 org.junit.jupiter:* = 5.0.2 org.mockito:* = 2.13.0 +org.slf4j:* = 1.7.25 From 5e7d4c0183fcf0ce38891010f939721252c163be Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Thu, 1 Feb 2018 23:17:33 -0500 Subject: [PATCH 02/11] wip --- grpc-json-bridge/build.gradle | 2 + .../CachingGrpcReflectionClientProvider.java | 36 ++++++++++ .../grpc/bridge/GrpcReflectionClientImpl.java | 61 ++++++++++------ .../bridge/ServerReflectionStubProvider.java | 13 ++++ .../bridge/GrpcBridgeImplIntegrationTest.java | 69 ++----------------- versions.props | 1 + 6 files changed, 96 insertions(+), 86 deletions(-) create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java diff --git a/grpc-json-bridge/build.gradle b/grpc-json-bridge/build.gradle index c0d49c8..c564c66 100644 --- a/grpc-json-bridge/build.gradle +++ b/grpc-json-bridge/build.gradle @@ -5,7 +5,9 @@ dependencies { compile 'io.grpc:grpc-netty' compile 'io.grpc:grpc-services' compile 'org.slf4j:slf4j-api' + compile 'com.github.ben-manes.caffeine:caffeine' testCompile project(':test-grpc-service') testCompile 'org.assertj:assertj-core' + testCompile 'org.slf4j:slf4j-simple' } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java new file mode 100644 index 0000000..08208e0 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java @@ -0,0 +1,36 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.protobuf.util.JsonFormat; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import javax.annotation.Nonnull; + +public class CachingGrpcReflectionClientProvider implements GrpcReflectionClient.Provider { + + private final ServerReflectionStubProvider stubProvider; + private final AsyncLoadingCache typeRegistries = Caffeine.newBuilder() + .buildAsync(new AsyncCacheLoader() { + @Nonnull + @Override + public CompletableFuture asyncLoad(String serviceName, Executor executor) { + stubProvider.get(serviceName) + return null; + } + }); + + public CachingGrpcReflectionClientProvider(ServerReflectionStubProvider stubProvider) { + this.stubProvider = stubProvider; + } + + @Override + public GrpcReflectionClient get(String serviceName) { + return () -> typeRegistries.get(serviceName); + } +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java index 3b74056..a364bdf 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,26 +34,24 @@ public class GrpcReflectionClientImpl implements GrpcReflectionClient { private final String serviceName; private final ServerReflectionGrpc.ServerReflectionStub reflection; - private StreamObserver reqStream; - public GrpcReflectionClientImpl(String serviceName, ServerReflectionGrpc.ServerReflectionStub reflection) { this.serviceName = serviceName; this.reflection = reflection; } - // TODO: thread safety?? caching?? @Override public CompletableFuture getTypeRegistry() { - CompletableFuture future = new CompletableFuture<>(); + CompletableFuture result = new CompletableFuture<>(); Set outstandingRequests = Sets.newHashSet(); Map protosByFileName = Maps.newHashMap(); - reqStream = reflection.serverReflectionInfo( + AtomicReference> reqStream = new AtomicReference<>(); + reqStream.set(reflection.serverReflectionInfo( new StreamObserver() { @Override public void onNext(ServerReflectionResponse response) { - if (future.isCancelled()) { + if (result.isCancelled()) { log.info("Future cancelled, not proceeding..."); return; } @@ -61,12 +60,18 @@ public void onNext(ServerReflectionResponse response) { switch (response.getMessageResponseCase()) { case LIST_SERVICES_RESPONSE: - requestAllFilesForServices(outstandingRequests, response); + requestAllFilesForServices(reqStream, outstandingRequests, response); + break; case FILE_DESCRIPTOR_RESPONSE: requestUnseenDependencyProtos( - outstandingRequests, protosByFileName, response.getFileDescriptorResponse()); + reqStream, + outstandingRequests, + protosByFileName, + response.getFileDescriptorResponse()); + break; default: log.error("Unexpected response case: {}", response.getMessageResponseCase()); + break; } if (outstandingRequests.isEmpty()) { @@ -75,7 +80,7 @@ public void onNext(ServerReflectionResponse response) { for (Descriptors.FileDescriptor compiledProto : compiledProtos) { typeRegistry.add(compiledProto.getMessageTypes()); } - future.complete(typeRegistry.build()); + result.complete(typeRegistry.build()); } } @@ -90,26 +95,32 @@ public void onError(Throwable error) { public void onCompleted() { log.info("Reflection complete, service {} likely shutting down.", serviceName); } - }); + })); // List all services to initial a download - reqStream.onNext(ServerReflectionRequest.newBuilder() + reqStream.get().onNext(ServerReflectionRequest.newBuilder() .setListServices("true") .build()); - return future; + return result; } private void requestAllFilesForServices( - Set outstandingRequests, ServerReflectionResponse response) { + AtomicReference> reqStream, + Set outstandingRequests, + ServerReflectionResponse response) { for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { - makeRequest(outstandingRequests, ServerReflectionRequest.newBuilder() - .setFileContainingSymbol(service.getName()) - .build()); + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileContainingSymbol(service.getName()) + .build()); } } private void requestUnseenDependencyProtos( + AtomicReference> reqStream, Set outstandingRequests, Map protosByFileName, FileDescriptorResponse response) { @@ -126,9 +137,12 @@ private void requestUnseenDependencyProtos( for (String dependencyFileName : protoDescriptor.getDependencyList()) { if (!protosByFileName.containsKey(dependencyFileName)) { - makeRequest(outstandingRequests, ServerReflectionRequest.newBuilder() - .setFileByFilename(dependencyFileName) - .build()); + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); } } } @@ -140,7 +154,7 @@ private Collection compileProtos( Map rootsByFileName = Maps.newHashMap(protosByFileName); for (DescriptorProtos.FileDescriptorProto proto : protosByFileName.values()) { for (String dependencyFileName : proto.getDependencyList()) { - protosByFileName.remove(dependencyFileName); + rootsByFileName.remove(dependencyFileName); } } @@ -183,9 +197,12 @@ public Iterable children( return compiledProtosByFileName.values(); } - private void makeRequest(Set outstandingRequests, ServerReflectionRequest request) { + private void makeRequest( + AtomicReference> reqStream, + Set outstandingRequests, + ServerReflectionRequest request) { outstandingRequests.add(request); - reqStream.onNext(request); + reqStream.get().onNext(request); } } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java new file mode 100644 index 0000000..1bdc3aa --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java @@ -0,0 +1,13 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; + +public interface ServerReflectionStubProvider { + + ServerReflectionGrpc.ServerReflectionStub get(String serviceName); + +} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java index f1a867a..a8c9e6c 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -5,11 +5,7 @@ package com.github.jared2501.grpc.bridge; import com.github.jared2501.grpc.bridge.test.TestServiceImpl; -import com.google.common.collect.Maps; -import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.ManagedChannel; import io.grpc.Server; @@ -18,10 +14,9 @@ import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.reflection.v1alpha.ServerReflectionGrpc; import io.grpc.reflection.v1alpha.ServerReflectionRequest; -import io.grpc.reflection.v1alpha.ServerReflectionResponse; import io.grpc.stub.StreamObserver; import java.io.IOException; -import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -53,64 +48,10 @@ void tearDown() throws InterruptedException { StreamObserver reqStream; @Test - void barbaz() throws InterruptedException, Descriptors.DescriptorValidationException { - ServerReflectionGrpc.ServerReflectionStub reflection = ServerReflectionGrpc.newStub(channel); - - // Steps: - // 1. List all services + kick of requests for their protos - // 2. For every proto get it's dependencies and if we don't have them then request them and record graph - // 3. Compile all protos - // 4. Build a type registry - // 5. Notify the consumer - - Map protoDescriptors = Maps.newHashMap(); - - reqStream = reflection.serverReflectionInfo( - new StreamObserver() { - @Override - public void onNext(ServerReflectionResponse value) { - System.out.println("all protos: " + protoDescriptors); - - for (ByteString protoBytes : value.getFileDescriptorResponse().getFileDescriptorProtoList()) { - DescriptorProtos.FileDescriptorProto protoDescriptor; - try { - protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - protoDescriptors.put(protoDescriptor.getName(), protoDescriptor); - - for (String dependencyFileName : protoDescriptor.getDependencyList()) { - if (!protoDescriptors.containsKey(dependencyFileName)) { - reqStream.onNext(ServerReflectionRequest.newBuilder() - .setFileByFilename(dependencyFileName) - .build()); - } - } - } - } - - @Override - public void onError(Throwable t) { - t.printStackTrace(); - } - - @Override - public void onCompleted() { - System.out.println("done!"); - } - }); - reqStream.onNext(ServerReflectionRequest.newBuilder() - .setFileContainingSymbol("com.github.jared2501.grpc.bridge.test.TestService") - .build()); - - Descriptors.FileDescriptor fileDescriptor = Descriptors.FileDescriptor.buildFrom(null, null); - JsonFormat.TypeRegistry.newBuilder() - .add(fileDescriptor.getMessageTypes()) - .build(); - - Thread.sleep(5000); + void barbaz() throws InterruptedException, Descriptors.DescriptorValidationException, ExecutionException { + GrpcReflectionClientImpl client = new GrpcReflectionClientImpl("foo", ServerReflectionGrpc.newStub(channel)); + JsonFormat.TypeRegistry typeRegistry = client.getTypeRegistry().get(); + System.out.println("types: " + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestService")); } } diff --git a/versions.props b/versions.props index edc30fb..f71f018 100644 --- a/versions.props +++ b/versions.props @@ -1,3 +1,4 @@ +com.github.ben-manes.caffeine:caffeine = 2.6.1 com.google.protobuf:* = 3.5.0 io.grpc:* = 1.7.1 org.apiguardian:apiguardian-api = 1.0.0 From 3eeacff2181822ceb3d269f173194f5b7d32e9e4 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Thu, 1 Feb 2018 23:36:34 -0500 Subject: [PATCH 03/11] wip --- .../CachingGrpcReflectionClientProvider.java | 36 ------ .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 12 +- ... ReflectionBasedTypeRegistrySupplier.java} | 107 +++++++++++------- ...nClient.java => TypeRegistrySupplier.java} | 8 +- .../bridge/GrpcBridgeImplIntegrationTest.java | 11 +- 5 files changed, 77 insertions(+), 97 deletions(-) delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java rename grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/{GrpcReflectionClientImpl.java => ReflectionBasedTypeRegistrySupplier.java} (62%) rename grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/{GrpcReflectionClient.java => TypeRegistrySupplier.java} (53%) diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java deleted file mode 100644 index 08208e0..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/CachingGrpcReflectionClientProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import com.github.benmanes.caffeine.cache.AsyncCacheLoader; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.protobuf.util.JsonFormat; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import javax.annotation.Nonnull; - -public class CachingGrpcReflectionClientProvider implements GrpcReflectionClient.Provider { - - private final ServerReflectionStubProvider stubProvider; - private final AsyncLoadingCache typeRegistries = Caffeine.newBuilder() - .buildAsync(new AsyncCacheLoader() { - @Nonnull - @Override - public CompletableFuture asyncLoad(String serviceName, Executor executor) { - stubProvider.get(serviceName) - return null; - } - }); - - public CachingGrpcReflectionClientProvider(ServerReflectionStubProvider stubProvider) { - this.stubProvider = stubProvider; - } - - @Override - public GrpcReflectionClient get(String serviceName) { - return () -> typeRegistries.get(serviceName); - } -} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index c6ead7b..e11672e 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -7,21 +7,19 @@ import com.google.protobuf.util.JsonFormat; import java.util.concurrent.CompletableFuture; -public final class GrpcBridgeImpl implements GrpcBridge { +final class GrpcBridgeImpl implements GrpcBridge { - private final GrpcReflectionClient.Provider clientProvider; + private final TypeRegistrySupplier typeRegistrySupplier; - public GrpcBridgeImpl(GrpcReflectionClient.Provider clientProvider) { - this.clientProvider = clientProvider; + public GrpcBridgeImpl(TypeRegistrySupplier typeRegistrySupplier) { + this.typeRegistrySupplier = typeRegistrySupplier; } @Override public CompletableFuture invoke(String serviceName, String method, byte[] jsonInput) { - return clientProvider.get(serviceName) - .getTypeRegistry() + return typeRegistrySupplier.getTypeRegistry(serviceName) .thenApply(typeRegistry -> { JsonFormat.parser().usingTypeRegistry(typeRegistry); - return null; }); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java similarity index 62% rename from grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java rename to grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java index a364bdf..9b9b490 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClientImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java @@ -4,12 +4,17 @@ package com.github.jared2501.grpc.bridge; +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.collect.TreeTraverser; import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos; -import com.google.protobuf.Descriptors; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.reflection.v1alpha.FileDescriptorResponse; @@ -22,29 +27,50 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class GrpcReflectionClientImpl implements GrpcReflectionClient { +/** + * A {@link TypeRegistrySupplier} that uses the {@link ServerReflectionGrpc ServerReflectionGrpc service} to build a + * {@link com.google.protobuf.util.JsonFormat.TypeRegistry} of types that are required to communicate with a specified + * service. + *

+ * TypeRegistries that are built will be cached for a specified period of time. + */ +public class ReflectionBasedTypeRegistrySupplier implements TypeRegistrySupplier { - private static final Logger log = LoggerFactory.getLogger(GrpcReflectionClientImpl.class); + private static final Logger log = LoggerFactory.getLogger(ReflectionBasedTypeRegistrySupplier.class); - private final String serviceName; - private final ServerReflectionGrpc.ServerReflectionStub reflection; + private final AsyncLoadingCache typeRegistries; - public GrpcReflectionClientImpl(String serviceName, ServerReflectionGrpc.ServerReflectionStub reflection) { - this.serviceName = serviceName; - this.reflection = reflection; + public ReflectionBasedTypeRegistrySupplier(ServerReflectionStubProvider stubProvider, Duration cacheDuration) { + this.typeRegistries = Caffeine.newBuilder() + .expireAfterWrite(cacheDuration.getNanos(), TimeUnit.NANOSECONDS) + .buildAsync(new AsyncCacheLoader() { + @Nonnull + @Override + public CompletableFuture asyncLoad(String serviceName, Executor executor) { + return getTypeRegistry(serviceName, stubProvider.get(serviceName)); + } + }); } @Override - public CompletableFuture getTypeRegistry() { + public CompletableFuture getTypeRegistry(String serviceName) { + return typeRegistries.get(serviceName); + } + + private static CompletableFuture getTypeRegistry( + String serviceName, ServerReflectionGrpc.ServerReflectionStub reflection) { CompletableFuture result = new CompletableFuture<>(); Set outstandingRequests = Sets.newHashSet(); - Map protosByFileName = Maps.newHashMap(); + Map protosByFileName = Maps.newHashMap(); AtomicReference> reqStream = new AtomicReference<>(); reqStream.set(reflection.serverReflectionInfo( @@ -60,11 +86,11 @@ public void onNext(ServerReflectionResponse response) { switch (response.getMessageResponseCase()) { case LIST_SERVICES_RESPONSE: - requestAllFilesForServices(reqStream, outstandingRequests, response); + requestAllFilesForServices(reqStream.get(), outstandingRequests, response); break; case FILE_DESCRIPTOR_RESPONSE: requestUnseenDependencyProtos( - reqStream, + reqStream.get(), outstandingRequests, protosByFileName, response.getFileDescriptorResponse()); @@ -75,9 +101,9 @@ public void onNext(ServerReflectionResponse response) { } if (outstandingRequests.isEmpty()) { - Collection compiledProtos = compileProtos(protosByFileName); + Collection compiledProtos = compileProtos(protosByFileName); JsonFormat.TypeRegistry.Builder typeRegistry = JsonFormat.TypeRegistry.newBuilder(); - for (Descriptors.FileDescriptor compiledProto : compiledProtos) { + for (FileDescriptor compiledProto : compiledProtos) { typeRegistry.add(compiledProto.getMessageTypes()); } result.complete(typeRegistry.build()); @@ -105,8 +131,8 @@ public void onCompleted() { return result; } - private void requestAllFilesForServices( - AtomicReference> reqStream, + private static void requestAllFilesForServices( + StreamObserver reqStream, Set outstandingRequests, ServerReflectionResponse response) { for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { @@ -119,15 +145,15 @@ private void requestAllFilesForServices( } } - private void requestUnseenDependencyProtos( - AtomicReference> reqStream, + private static void requestUnseenDependencyProtos( + StreamObserver reqStream, Set outstandingRequests, - Map protosByFileName, + Map protosByFileName, FileDescriptorResponse response) { for (ByteString protoBytes : response.getFileDescriptorProtoList()) { - DescriptorProtos.FileDescriptorProto protoDescriptor; + FileDescriptorProto protoDescriptor; try { - protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); + protoDescriptor = FileDescriptorProto.parseFrom(protoBytes); } catch (InvalidProtocolBufferException e) { log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); continue; @@ -148,25 +174,24 @@ private void requestUnseenDependencyProtos( } } - private Collection compileProtos( - Map protosByFileName) { + private static Collection compileProtos(Map protosByFileName) { // Find all "roots", where a root is a proto file for which another proto file does not depend on it - Map rootsByFileName = Maps.newHashMap(protosByFileName); - for (DescriptorProtos.FileDescriptorProto proto : protosByFileName.values()) { + Map rootsByFileName = Maps.newHashMap(protosByFileName); + for (FileDescriptorProto proto : protosByFileName.values()) { for (String dependencyFileName : proto.getDependencyList()) { rootsByFileName.remove(dependencyFileName); } } - Map compiledProtosByFileName = Maps.newHashMap(); + Map compiledProtosByFileName = Maps.newHashMap(); // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto // file if it has not already been compiled - TreeTraverser treeTraverser = - new TreeTraverser() { + TreeTraverser treeTraverser = + new TreeTraverser() { @Override - public Iterable children( - DescriptorProtos.FileDescriptorProto root) { + public Iterable children( + FileDescriptorProto root) { return root.getDependencyList() .stream() // Note: skip visiting dependencies if they have already been compiled @@ -175,17 +200,17 @@ public Iterable children( .collect(Collectors.toSet()); } }; - for (DescriptorProtos.FileDescriptorProto root : rootsByFileName.values()) { - for (DescriptorProtos.FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { - Descriptors.FileDescriptor[] dependencies = proto.getDependencyList() + for (FileDescriptorProto root : rootsByFileName.values()) { + for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + FileDescriptor[] dependencies = proto.getDependencyList() .stream() .map(compiledProtosByFileName::get) - .toArray(Descriptors.FileDescriptor[]::new); + .toArray(FileDescriptor[]::new); - Descriptors.FileDescriptor compiledProto; + FileDescriptor compiledProto; try { - compiledProto = Descriptors.FileDescriptor.buildFrom(proto, dependencies); - } catch (Descriptors.DescriptorValidationException e) { + compiledProto = FileDescriptor.buildFrom(proto, dependencies); + } catch (DescriptorValidationException e) { log.warn("Exception encountered when building proto... skipping", e); continue; } @@ -197,12 +222,12 @@ public Iterable children( return compiledProtosByFileName.values(); } - private void makeRequest( - AtomicReference> reqStream, + private static void makeRequest( + StreamObserver reqStream, Set outstandingRequests, ServerReflectionRequest request) { outstandingRequests.add(request); - reqStream.get().onNext(request); + reqStream.onNext(request); } } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java similarity index 53% rename from grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java rename to grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java index a7c18ed..cc0ed05 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcReflectionClient.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java @@ -7,12 +7,8 @@ import com.google.protobuf.util.JsonFormat; import java.util.concurrent.CompletableFuture; -public interface GrpcReflectionClient { +interface TypeRegistrySupplier { - interface Provider { - GrpcReflectionClient get(String serviceName); - } - - CompletableFuture getTypeRegistry(); + CompletableFuture getTypeRegistry(String serviceName); } diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java index a8c9e6c..0a132be 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -13,8 +13,6 @@ import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.reflection.v1alpha.ServerReflectionGrpc; -import io.grpc.reflection.v1alpha.ServerReflectionRequest; -import io.grpc.stub.StreamObserver; import java.io.IOException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -45,13 +43,12 @@ void tearDown() throws InterruptedException { server.awaitTermination(); } - StreamObserver reqStream; - @Test void barbaz() throws InterruptedException, Descriptors.DescriptorValidationException, ExecutionException { - GrpcReflectionClientImpl client = new GrpcReflectionClientImpl("foo", ServerReflectionGrpc.newStub(channel)); - JsonFormat.TypeRegistry typeRegistry = client.getTypeRegistry().get(); - System.out.println("types: " + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestService")); + ReflectionBasedTypeRegistrySupplier clientProvider = new ReflectionBasedTypeRegistrySupplier( + serviceName -> ServerReflectionGrpc.newStub(channel)); + JsonFormat.TypeRegistry typeRegistry = clientProvider.getTypeRegistry("foo").get(); + System.out.println("types: " + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestMessage")); } } From 5f0c55531e3392287ac1f6700f366503f799ad42 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Fri, 2 Feb 2018 14:14:01 -0500 Subject: [PATCH 04/11] wip --- .../grpc/bridge/ChannelProvider.java | 13 ++ .../jared2501/grpc/bridge/GrpcBridge.java | 3 +- .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 111 +++++++++++++++++- .../ReflectionBasedTypeRegistrySupplier.java | 11 +- .../bridge/ServerReflectionStubProvider.java | 13 -- .../bridge/GrpcBridgeImplIntegrationTest.java | 19 ++- 6 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java new file mode 100644 index 0000000..6111cc5 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java @@ -0,0 +1,13 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import io.grpc.Channel; + +public interface ChannelProvider { + + Channel get(String serviceName); + +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java index f59b64d..80b9c18 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java @@ -4,7 +4,6 @@ package com.github.jared2501.grpc.bridge; -import com.google.common.util.concurrent.ListenableFuture; import java.util.concurrent.CompletableFuture; public interface GrpcBridge { @@ -12,5 +11,5 @@ public interface GrpcBridge { * Invokes the specified method on the specified service with the given JSON input, returns the JSON output of the * result. Note that streaming methods will result in an error state for the returned future. */ - CompletableFuture invoke(String serviceName, String method, byte[] jsonInput); + CompletableFuture invoke(String serviceName, String method, String jsonInput); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index e11672e..2016c38 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -4,24 +4,125 @@ package com.github.jared2501.grpc.bridge; +import com.google.common.io.ByteStreams; +import com.google.protobuf.Any; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Empty; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import io.grpc.CallOptions; +import io.grpc.ClientCall; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Status; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.CompletableFuture; final class GrpcBridgeImpl implements GrpcBridge { + private final ChannelProvider channelProvider; private final TypeRegistrySupplier typeRegistrySupplier; - public GrpcBridgeImpl(TypeRegistrySupplier typeRegistrySupplier) { + public GrpcBridgeImpl(ChannelProvider channelProvider, TypeRegistrySupplier typeRegistrySupplier) { + this.channelProvider = channelProvider; this.typeRegistrySupplier = typeRegistrySupplier; } @Override - public CompletableFuture invoke(String serviceName, String method, byte[] jsonInput) { + public CompletableFuture invoke(String serviceName, String fullMethodName, String jsonInput) { return typeRegistrySupplier.getTypeRegistry(serviceName) - .thenApply(typeRegistry -> { - JsonFormat.parser().usingTypeRegistry(typeRegistry); - return null; + .thenComposeAsync(typeRegistry -> { + ClientCall call = channelProvider.get(serviceName) + .newCall(getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); + + CompletableFuture future = new CompletableFuture<>(); + + // TODO: handle cancellations? maybe futures not right abstraction? + call.start( + new ClientCall.Listener() { + @Override + public void onHeaders(Metadata headers) { + System.out.println("headers: " + headers); + } + + @Override + public void onMessage(byte[] messageBytes) { + DynamicMessage message; + try { + message = DynamicMessage.parseFrom(typeRegistry.find( + "com.github.jared2501.grpc.bridge.test.TestMessage"), + messageBytes); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + return; + } + + try { + JsonFormat.printer().usingTypeRegistry(typeRegistry).appendTo(message, System.out); + } catch (IOException e) { + e.printStackTrace(); + return; + } + } + + @Override + public void onClose(Status status, Metadata trailers) { + System.out.println("closed! " + status); + } + + @Override + public void onReady() { + } + }, + new Metadata()); + + DynamicMessage.Builder message = DynamicMessage.newBuilder( + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestMessage")); + JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry); + try { + parser.merge(jsonInput, message); + } catch (InvalidProtocolBufferException e) { + future.obtrudeException(new RuntimeException( + "InvalidProtocolBufferException encountered when converting JSON to proto", e)); + return future; + } + + call.sendMessage(message.build().toByteArray()); + call.halfClose(); + call.request(2); + + return future; }); } + private MethodDescriptor getMethodDescriptor(String fullMethodName) { + return MethodDescriptor.newBuilder(ByteMarshaller.INSTANCE, ByteMarshaller.INSTANCE) + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName(fullMethodName) + .setIdempotent(false) + .setSafe(false) + .build(); + } + + private static class ByteMarshaller implements MethodDescriptor.Marshaller { + static final ByteMarshaller INSTANCE = new ByteMarshaller(); + + @Override + public InputStream stream(byte[] value) { + return new ByteArrayInputStream(value); + } + + @Override + public byte[] parse(InputStream stream) { + try { + return ByteStreams.toByteArray(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java index 9b9b490..ac9c9ad 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java @@ -14,7 +14,6 @@ import com.google.protobuf.DescriptorProtos.FileDescriptorProto; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Descriptors.FileDescriptor; -import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.reflection.v1alpha.FileDescriptorResponse; @@ -42,20 +41,22 @@ *

* TypeRegistries that are built will be cached for a specified period of time. */ -public class ReflectionBasedTypeRegistrySupplier implements TypeRegistrySupplier { +class ReflectionBasedTypeRegistrySupplier implements TypeRegistrySupplier { private static final Logger log = LoggerFactory.getLogger(ReflectionBasedTypeRegistrySupplier.class); private final AsyncLoadingCache typeRegistries; - public ReflectionBasedTypeRegistrySupplier(ServerReflectionStubProvider stubProvider, Duration cacheDuration) { + ReflectionBasedTypeRegistrySupplier( + ChannelProvider stubProvider, long expiryDuration, TimeUnit expiryTimeUnit) { this.typeRegistries = Caffeine.newBuilder() - .expireAfterWrite(cacheDuration.getNanos(), TimeUnit.NANOSECONDS) + .expireAfterWrite(expiryDuration, expiryTimeUnit) .buildAsync(new AsyncCacheLoader() { @Nonnull @Override public CompletableFuture asyncLoad(String serviceName, Executor executor) { - return getTypeRegistry(serviceName, stubProvider.get(serviceName)); + return getTypeRegistry( + serviceName, ServerReflectionGrpc.newStub(stubProvider.get(serviceName))); } }); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java deleted file mode 100644 index 1bdc3aa..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServerReflectionStubProvider.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; - -public interface ServerReflectionStubProvider { - - ServerReflectionGrpc.ServerReflectionStub get(String serviceName); - -} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java index 0a132be..16334f0 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -5,7 +5,6 @@ package com.github.jared2501.grpc.bridge; import com.github.jared2501.grpc.bridge.test.TestServiceImpl; -import com.google.protobuf.Descriptors; import com.google.protobuf.util.JsonFormat; import io.grpc.ManagedChannel; import io.grpc.Server; @@ -14,6 +13,7 @@ import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.reflection.v1alpha.ServerReflectionGrpc; import java.io.IOException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; @@ -44,11 +44,24 @@ void tearDown() throws InterruptedException { } @Test - void barbaz() throws InterruptedException, Descriptors.DescriptorValidationException, ExecutionException { + void barbaz() throws InterruptedException, ExecutionException { ReflectionBasedTypeRegistrySupplier clientProvider = new ReflectionBasedTypeRegistrySupplier( - serviceName -> ServerReflectionGrpc.newStub(channel)); + serviceName -> channel, 1, TimeUnit.NANOSECONDS); JsonFormat.TypeRegistry typeRegistry = clientProvider.getTypeRegistry("foo").get(); System.out.println("types: " + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestMessage")); } + @Test + void norfquix() throws ExecutionException, InterruptedException { + ChannelProvider channelProvider = serviceName -> channel; + GrpcBridgeImpl bridge = new GrpcBridgeImpl( + channelProvider, new ReflectionBasedTypeRegistrySupplier(channelProvider, 1, TimeUnit.NANOSECONDS)); + + CompletableFuture result = bridge.invoke( + "foo", + "com.github.jared2501.grpc.bridge.test.TestService/UnaryReqUnaryResp", + "{\"message\": \"message\", \"empty\": {}}"); + + result.get(); + } } From bfec8e51922e789ac13da87f3156f27fb8d23ad6 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Fri, 2 Feb 2018 15:19:06 -0500 Subject: [PATCH 05/11] After the refactor... not sure if better? --- .../jared2501/grpc/bridge/GrpcBridge.java | 15 +- .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 67 +++-- .../ReflectionBasedTypeRegistrySupplier.java | 234 ----------------- .../bridge/ReflectionResponseObserver.java | 242 ++++++++++++++++++ .../grpc/bridge/TypeRegistrySupplier.java | 14 - .../bridge/GrpcBridgeImplIntegrationTest.java | 39 +-- 6 files changed, 316 insertions(+), 295 deletions(-) delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java index 80b9c18..d0dc7e6 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java @@ -4,12 +4,21 @@ package com.github.jared2501.grpc.bridge; -import java.util.concurrent.CompletableFuture; - public interface GrpcBridge { + interface InvocationHandle { + void start(); + void cancel(); + } + + interface InvocationObserver { + void onResult(String jsonOutput); + void onMethodNotFound(); + void onError(Throwable error); + } + /** * Invokes the specified method on the specified service with the given JSON input, returns the JSON output of the * result. Note that streaming methods will result in an error state for the returned future. */ - CompletableFuture invoke(String serviceName, String method, String jsonInput); + InvocationHandle invoke(String serviceName, String fullMethodName, String jsonInput, InvocationObserver observer); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index 2016c38..9090bda 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -5,42 +5,48 @@ package com.github.jared2501.grpc.bridge; import com.google.common.io.ByteStreams; -import com.google.protobuf.Any; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; -import com.google.protobuf.Empty; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.CallOptions; +import io.grpc.Channel; import io.grpc.ClientCall; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.Status; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.concurrent.CompletableFuture; +import java.util.Optional; final class GrpcBridgeImpl implements GrpcBridge { private final ChannelProvider channelProvider; - private final TypeRegistrySupplier typeRegistrySupplier; - public GrpcBridgeImpl(ChannelProvider channelProvider, TypeRegistrySupplier typeRegistrySupplier) { + public GrpcBridgeImpl(ChannelProvider channelProvider) { this.channelProvider = channelProvider; - this.typeRegistrySupplier = typeRegistrySupplier; } @Override - public CompletableFuture invoke(String serviceName, String fullMethodName, String jsonInput) { - return typeRegistrySupplier.getTypeRegistry(serviceName) - .thenComposeAsync(typeRegistry -> { + public InvocationHandle invoke( + String serviceName, String fullMethodName, String jsonInput, InvocationObserver observer) { + return new InvocationHandle() { + @Override + public void start() { + Channel channel = channelProvider.get(serviceName); + ReflectionResponseObserver reflection = new ReflectionResponseObserver(serviceName); + reflection.onDoneHandler(() -> { + Optional maybeMethod = reflection.getAvailableMethod(fullMethodName); + if (!maybeMethod.isPresent()) { + observer.onMethodNotFound(); + return; + } + Descriptors.MethodDescriptor method = maybeMethod.get(); + ClientCall call = channelProvider.get(serviceName) .newCall(getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); - - CompletableFuture future = new CompletableFuture<>(); - - // TODO: handle cancellations? maybe futures not right abstraction? call.start( new ClientCall.Listener() { @Override @@ -52,25 +58,27 @@ public void onHeaders(Metadata headers) { public void onMessage(byte[] messageBytes) { DynamicMessage message; try { - message = DynamicMessage.parseFrom(typeRegistry.find( - "com.github.jared2501.grpc.bridge.test.TestMessage"), - messageBytes); + message = DynamicMessage.parseFrom(method.getOutputType(), messageBytes); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + observer.onError(new RuntimeException( + "InvalidProtocolBufferException encountered when parsing message", e)); return; } try { - JsonFormat.printer().usingTypeRegistry(typeRegistry).appendTo(message, System.out); + JsonFormat.printer() + .usingTypeRegistry(reflection.getTypeRegistry()) + .appendTo(message, System.out); } catch (IOException e) { - e.printStackTrace(); - return; + observer.onError(new RuntimeException( + "IOException encountered when printing response", e)); } } @Override public void onClose(Status status, Metadata trailers) { System.out.println("closed! " + status); + reflection.cancel(); } @Override @@ -79,23 +87,28 @@ public void onReady() { }, new Metadata()); - DynamicMessage.Builder message = DynamicMessage.newBuilder( - typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestMessage")); - JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry); + DynamicMessage.Builder message = DynamicMessage.newBuilder(method.getInputType()); + JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(reflection.getTypeRegistry()); try { parser.merge(jsonInput, message); } catch (InvalidProtocolBufferException e) { - future.obtrudeException(new RuntimeException( + observer.onError(new RuntimeException( "InvalidProtocolBufferException encountered when converting JSON to proto", e)); - return future; + return; } call.sendMessage(message.build().toByteArray()); call.halfClose(); call.request(2); - - return future; }); + reflection.start(ServerReflectionGrpc.newStub(channel)); + } + + @Override + public void cancel() { + // TODO(jnewman): implement! + } + }; } private MethodDescriptor getMethodDescriptor(String fullMethodName) { diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java deleted file mode 100644 index ac9c9ad..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionBasedTypeRegistrySupplier.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import com.github.benmanes.caffeine.cache.AsyncCacheLoader; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.collect.TreeTraverser; -import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos.FileDescriptorProto; -import com.google.protobuf.Descriptors.DescriptorValidationException; -import com.google.protobuf.Descriptors.FileDescriptor; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import io.grpc.reflection.v1alpha.FileDescriptorResponse; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; -import io.grpc.reflection.v1alpha.ServerReflectionRequest; -import io.grpc.reflection.v1alpha.ServerReflectionResponse; -import io.grpc.reflection.v1alpha.ServiceResponse; -import io.grpc.stub.StreamObserver; -import java.util.Collection; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A {@link TypeRegistrySupplier} that uses the {@link ServerReflectionGrpc ServerReflectionGrpc service} to build a - * {@link com.google.protobuf.util.JsonFormat.TypeRegistry} of types that are required to communicate with a specified - * service. - *

- * TypeRegistries that are built will be cached for a specified period of time. - */ -class ReflectionBasedTypeRegistrySupplier implements TypeRegistrySupplier { - - private static final Logger log = LoggerFactory.getLogger(ReflectionBasedTypeRegistrySupplier.class); - - private final AsyncLoadingCache typeRegistries; - - ReflectionBasedTypeRegistrySupplier( - ChannelProvider stubProvider, long expiryDuration, TimeUnit expiryTimeUnit) { - this.typeRegistries = Caffeine.newBuilder() - .expireAfterWrite(expiryDuration, expiryTimeUnit) - .buildAsync(new AsyncCacheLoader() { - @Nonnull - @Override - public CompletableFuture asyncLoad(String serviceName, Executor executor) { - return getTypeRegistry( - serviceName, ServerReflectionGrpc.newStub(stubProvider.get(serviceName))); - } - }); - } - - @Override - public CompletableFuture getTypeRegistry(String serviceName) { - return typeRegistries.get(serviceName); - } - - private static CompletableFuture getTypeRegistry( - String serviceName, ServerReflectionGrpc.ServerReflectionStub reflection) { - CompletableFuture result = new CompletableFuture<>(); - - Set outstandingRequests = Sets.newHashSet(); - Map protosByFileName = Maps.newHashMap(); - - AtomicReference> reqStream = new AtomicReference<>(); - reqStream.set(reflection.serverReflectionInfo( - new StreamObserver() { - @Override - public void onNext(ServerReflectionResponse response) { - if (result.isCancelled()) { - log.info("Future cancelled, not proceeding..."); - return; - } - - outstandingRequests.remove(response.getOriginalRequest()); - - switch (response.getMessageResponseCase()) { - case LIST_SERVICES_RESPONSE: - requestAllFilesForServices(reqStream.get(), outstandingRequests, response); - break; - case FILE_DESCRIPTOR_RESPONSE: - requestUnseenDependencyProtos( - reqStream.get(), - outstandingRequests, - protosByFileName, - response.getFileDescriptorResponse()); - break; - default: - log.error("Unexpected response case: {}", response.getMessageResponseCase()); - break; - } - - if (outstandingRequests.isEmpty()) { - Collection compiledProtos = compileProtos(protosByFileName); - JsonFormat.TypeRegistry.Builder typeRegistry = JsonFormat.TypeRegistry.newBuilder(); - for (FileDescriptor compiledProto : compiledProtos) { - typeRegistry.add(compiledProto.getMessageTypes()); - } - result.complete(typeRegistry.build()); - } - } - - @Override - public void onError(Throwable error) { - log.error( - "Throwable encountered when streaming handling reflecting for service {}", - serviceName, error); - } - - @Override - public void onCompleted() { - log.info("Reflection complete, service {} likely shutting down.", serviceName); - } - })); - - // List all services to initial a download - reqStream.get().onNext(ServerReflectionRequest.newBuilder() - .setListServices("true") - .build()); - - return result; - } - - private static void requestAllFilesForServices( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionResponse response) { - for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileContainingSymbol(service.getName()) - .build()); - } - } - - private static void requestUnseenDependencyProtos( - StreamObserver reqStream, - Set outstandingRequests, - Map protosByFileName, - FileDescriptorResponse response) { - for (ByteString protoBytes : response.getFileDescriptorProtoList()) { - FileDescriptorProto protoDescriptor; - try { - protoDescriptor = FileDescriptorProto.parseFrom(protoBytes); - } catch (InvalidProtocolBufferException e) { - log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); - continue; - } - - protosByFileName.put(protoDescriptor.getName(), protoDescriptor); - - for (String dependencyFileName : protoDescriptor.getDependencyList()) { - if (!protosByFileName.containsKey(dependencyFileName)) { - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileByFilename(dependencyFileName) - .build()); - } - } - } - } - - private static Collection compileProtos(Map protosByFileName) { - // Find all "roots", where a root is a proto file for which another proto file does not depend on it - Map rootsByFileName = Maps.newHashMap(protosByFileName); - for (FileDescriptorProto proto : protosByFileName.values()) { - for (String dependencyFileName : proto.getDependencyList()) { - rootsByFileName.remove(dependencyFileName); - } - } - - Map compiledProtosByFileName = Maps.newHashMap(); - - // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto - // file if it has not already been compiled - TreeTraverser treeTraverser = - new TreeTraverser() { - @Override - public Iterable children( - FileDescriptorProto root) { - return root.getDependencyList() - .stream() - // Note: skip visiting dependencies if they have already been compiled - .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) - .map(protosByFileName::get) - .collect(Collectors.toSet()); - } - }; - for (FileDescriptorProto root : rootsByFileName.values()) { - for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { - FileDescriptor[] dependencies = proto.getDependencyList() - .stream() - .map(compiledProtosByFileName::get) - .toArray(FileDescriptor[]::new); - - FileDescriptor compiledProto; - try { - compiledProto = FileDescriptor.buildFrom(proto, dependencies); - } catch (DescriptorValidationException e) { - log.warn("Exception encountered when building proto... skipping", e); - continue; - } - - compiledProtosByFileName.put(proto.getName(), compiledProto); - } - } - - return compiledProtosByFileName.values(); - } - - private static void makeRequest( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionRequest request) { - outstandingRequests.add(request); - reqStream.onNext(request); - } - -} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java new file mode 100644 index 0000000..f3e4901 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java @@ -0,0 +1,242 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeTraverser; +import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.Descriptors.MethodDescriptor; +import com.google.protobuf.Descriptors.ServiceDescriptor; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import io.grpc.Context; +import io.grpc.reflection.v1alpha.FileDescriptorResponse; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; +import io.grpc.reflection.v1alpha.ServerReflectionRequest; +import io.grpc.reflection.v1alpha.ServerReflectionResponse; +import io.grpc.reflection.v1alpha.ServiceResponse; +import io.grpc.stub.StreamObserver; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ReflectionResponseObserver implements StreamObserver { + + private static final Logger log = LoggerFactory.getLogger(ReflectionResponseObserver.class); + + private final Set outstandingRequests = Sets.newHashSet(); + private final Map protosByFileName = Maps.newHashMap(); + private final String serviceName; + + private Runnable onDoneHandler; + private StreamObserver reqStream; + private Context.CancellableContext context; + + private final Set availableServices = Sets.newHashSet(); + private final Map availableMethodsByFullMethodName = Maps.newHashMap(); + private JsonFormat.TypeRegistry typeRegistry; + + public ReflectionResponseObserver(String serviceName) { + this.serviceName = serviceName; + } + + public void start(ServerReflectionGrpc.ServerReflectionStub serverReflection) { + context = Context.CancellableContext.current().withCancellation(); + context.run(() -> reqStream = serverReflection.serverReflectionInfo(this)); + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setListServices("true") + .build()); + } + + public void onDoneHandler(Runnable runnable) { + this.onDoneHandler = runnable; + } + + public void cancel() { + context.cancel(new RuntimeException()); + } + + public JsonFormat.TypeRegistry getTypeRegistry() { + checkState(typeRegistry != null, "ReflectionResponseObserver has not finished yet"); + return typeRegistry; + } + + public Optional getAvailableMethod(String fullMethodName) { + checkState(typeRegistry != null, "ReflectionResponseObserver has not finished yet"); + return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); + } + + @Override + public void onNext(ServerReflectionResponse response) { + if (context.isCancelled()) { + log.info("Future cancelled, not proceeding..."); + return; + } + + outstandingRequests.remove(response.getOriginalRequest()); + + switch (response.getMessageResponseCase()) { + case LIST_SERVICES_RESPONSE: + requestAllFilesForServices(reqStream, outstandingRequests, response); + break; + case FILE_DESCRIPTOR_RESPONSE: + requestUnseenDependencyProtos( + reqStream, + outstandingRequests, + protosByFileName, + response.getFileDescriptorResponse()); + break; + default: + log.error("Unexpected response case: {}", response.getMessageResponseCase()); + break; + } + + if (outstandingRequests.isEmpty()) { + JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); + + Collection compiledProtos = compileProtos(protosByFileName); + for (FileDescriptor compiledProto : compiledProtos) { + typeRegistryBuilder.add(compiledProto.getMessageTypes()); + for (ServiceDescriptor service : compiledProto.getServices()) { + if (availableServices.contains(service.getFullName())) { + for (MethodDescriptor method : service.getMethods()) { + String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); + availableMethodsByFullMethodName.put(methodFullName, method); + } + } + } + } + + typeRegistry = typeRegistryBuilder.build(); + + onDoneHandler.run(); + } + } + + @Override + public void onError(Throwable error) { + log.error( + "Throwable encountered when streaming handling reflecting for service {}", + serviceName, error); + // TODO(jnewman): handle errors here better + } + + @Override + public void onCompleted() { + log.info("Reflection complete, service {} likely shutting down.", serviceName); + // TODO(jnewman): handle this better + } + + private void requestAllFilesForServices( + StreamObserver reqStream, + Set outstandingRequests, + ServerReflectionResponse response) { + for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { + availableServices.add(service.getName()); + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileContainingSymbol(service.getName()) + .build()); + } + } + + private static void requestUnseenDependencyProtos( + StreamObserver reqStream, + Set outstandingRequests, + Map protosByFileName, + FileDescriptorResponse response) { + for (ByteString protoBytes : response.getFileDescriptorProtoList()) { + FileDescriptorProto protoDescriptor; + try { + protoDescriptor = FileDescriptorProto.parseFrom(protoBytes); + } catch (InvalidProtocolBufferException e) { + log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); + continue; + } + + protosByFileName.put(protoDescriptor.getName(), protoDescriptor); + + for (String dependencyFileName : protoDescriptor.getDependencyList()) { + if (!protosByFileName.containsKey(dependencyFileName)) { + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); + } + } + } + } + + private static Collection compileProtos( + Map protosByFileName) { + // Find all "roots", where a root is a proto file for which another proto file does not depend on it + Map rootsByFileName = Maps.newHashMap(protosByFileName); + for (FileDescriptorProto proto : protosByFileName.values()) { + for (String dependencyFileName : proto.getDependencyList()) { + rootsByFileName.remove(dependencyFileName); + } + } + + Map compiledProtosByFileName = Maps.newHashMap(); + + // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto + // file if it has not already been compiled + TreeTraverser treeTraverser = new TreeTraverser() { + @Override + public Iterable children( + FileDescriptorProto root) { + return root.getDependencyList() + .stream() + // Note: skip visiting dependencies if they have already been compiled + .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) + .map(protosByFileName::get) + .collect(Collectors.toSet()); + } + }; + for (FileDescriptorProto root : rootsByFileName.values()) { + for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + FileDescriptor[] dependencies = proto.getDependencyList() + .stream() + .map(compiledProtosByFileName::get) + .toArray(FileDescriptor[]::new); + + FileDescriptor compiledProto; + try { + compiledProto = FileDescriptor.buildFrom(proto, dependencies); + } catch (DescriptorValidationException e) { + log.warn("Exception encountered when building proto... skipping", e); + continue; + } + + compiledProtosByFileName.put(proto.getName(), compiledProto); + } + } + + return compiledProtosByFileName.values(); + } + + private static void makeRequest( + StreamObserver reqStream, + Set outstandingRequests, + ServerReflectionRequest request) { + outstandingRequests.add(request); + reqStream.onNext(request); + } + +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java deleted file mode 100644 index cc0ed05..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/TypeRegistrySupplier.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import com.google.protobuf.util.JsonFormat; -import java.util.concurrent.CompletableFuture; - -interface TypeRegistrySupplier { - - CompletableFuture getTypeRegistry(String serviceName); - -} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java index 16334f0..581d13d 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -5,15 +5,12 @@ package com.github.jared2501.grpc.bridge; import com.github.jared2501.grpc.bridge.test.TestServiceImpl; -import com.google.protobuf.util.JsonFormat; import io.grpc.ManagedChannel; import io.grpc.Server; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; import io.grpc.protobuf.services.ProtoReflectionService; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; import java.io.IOException; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; @@ -43,25 +40,33 @@ void tearDown() throws InterruptedException { server.awaitTermination(); } - @Test - void barbaz() throws InterruptedException, ExecutionException { - ReflectionBasedTypeRegistrySupplier clientProvider = new ReflectionBasedTypeRegistrySupplier( - serviceName -> channel, 1, TimeUnit.NANOSECONDS); - JsonFormat.TypeRegistry typeRegistry = clientProvider.getTypeRegistry("foo").get(); - System.out.println("types: " + typeRegistry.find("com.github.jared2501.grpc.bridge.test.TestMessage")); - } - @Test void norfquix() throws ExecutionException, InterruptedException { - ChannelProvider channelProvider = serviceName -> channel; - GrpcBridgeImpl bridge = new GrpcBridgeImpl( - channelProvider, new ReflectionBasedTypeRegistrySupplier(channelProvider, 1, TimeUnit.NANOSECONDS)); + GrpcBridge bridge = new GrpcBridgeImpl(serviceName -> channel); - CompletableFuture result = bridge.invoke( + GrpcBridge.InvocationHandle handle = bridge.invoke( "foo", "com.github.jared2501.grpc.bridge.test.TestService/UnaryReqUnaryResp", - "{\"message\": \"message\", \"empty\": {}}"); + "{\"message\": \"message\", \"empty\": {}}", + new GrpcBridge.InvocationObserver() { + @Override + public void onResult(String jsonOutput) { + System.out.println("output: " + jsonOutput); + } + + @Override + public void onMethodNotFound() { + System.out.println("method not found!"); + } + + @Override + public void onError(Throwable error) { + error.printStackTrace(); + } + }); + + handle.start(); - result.get(); + Thread.sleep(5000); } } From 061c192316073cefa40e431ec975d0483cfc1760 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Tue, 6 Feb 2018 18:51:40 -0800 Subject: [PATCH 06/11] wip --- .../com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java | 4 ++-- .../jared2501/grpc/bridge/ReflectionResponseObserver.java | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index 9090bda..f4a32ce 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -45,8 +45,8 @@ public void start() { } Descriptors.MethodDescriptor method = maybeMethod.get(); - ClientCall call = channelProvider.get(serviceName) - .newCall(getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); + ClientCall call = channel.newCall( + getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); call.start( new ClientCall.Listener() { @Override diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java index f3e4901..6b41c78 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java @@ -105,7 +105,6 @@ public void onNext(ServerReflectionResponse response) { if (outstandingRequests.isEmpty()) { JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); - Collection compiledProtos = compileProtos(protosByFileName); for (FileDescriptor compiledProto : compiledProtos) { typeRegistryBuilder.add(compiledProto.getMessageTypes()); @@ -118,9 +117,7 @@ public void onNext(ServerReflectionResponse response) { } } } - typeRegistry = typeRegistryBuilder.build(); - onDoneHandler.run(); } } @@ -199,8 +196,7 @@ private static Collection compileProtos( // file if it has not already been compiled TreeTraverser treeTraverser = new TreeTraverser() { @Override - public Iterable children( - FileDescriptorProto root) { + public Iterable children(FileDescriptorProto root) { return root.getDependencyList() .stream() // Note: skip visiting dependencies if they have already been compiled From 3af7f4cc79e7437df8527a7f7aa7b93570b42e77 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Wed, 7 Feb 2018 16:28:00 -0800 Subject: [PATCH 07/11] dw --- grpc-json-bridge-server/build.gradle | 6 ++++++ settings.gradle | 1 + versions.props | 1 + 3 files changed, 8 insertions(+) create mode 100644 grpc-json-bridge-server/build.gradle diff --git a/grpc-json-bridge-server/build.gradle b/grpc-json-bridge-server/build.gradle new file mode 100644 index 0000000..59c5420 --- /dev/null +++ b/grpc-json-bridge-server/build.gradle @@ -0,0 +1,6 @@ +apply from: rootProject.file('gradle/java.gradle') +apply from: rootProject.file('gradle/junit5.gradle') + +dependencies { + compile 'io.dropwizard:dropwizard-core' +} diff --git a/settings.gradle b/settings.gradle index 9747cca..f5796e5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ rootProject.name = 'grpc-json-bridge' include 'grpc-json-bridge' +include 'grpc-json-bridge-server' include 'test-grpc-service' diff --git a/versions.props b/versions.props index f71f018..f60da18 100644 --- a/versions.props +++ b/versions.props @@ -1,5 +1,6 @@ com.github.ben-manes.caffeine:caffeine = 2.6.1 com.google.protobuf:* = 3.5.0 +io.dropwizard:* = 1.2.3 io.grpc:* = 1.7.1 org.apiguardian:apiguardian-api = 1.0.0 org.assertj:assertj-core = 3.6.2 From 834dee21de8a0a31938fb9f4270f539b6451f3bf Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Wed, 14 Feb 2018 11:43:20 -0800 Subject: [PATCH 08/11] wip2 --- grpc-json-bridge/build.gradle | 2 + .../grpc/bridge/AvailableMethod.java | 30 ++ .../grpc/bridge/ChannelProvider.java | 13 - .../jared2501/grpc/bridge/GrpcBridge.java | 13 +- .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 309 +++++++++++++++--- .../bridge/ReflectionResponseObserver.java | 238 -------------- .../jared2501/grpc/bridge/ServiceIndex.java | 30 ++ .../grpc/bridge/ServiceIndexImpl.java | 135 ++++++++ versions.props | 1 + 9 files changed, 467 insertions(+), 304 deletions(-) create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java diff --git a/grpc-json-bridge/build.gradle b/grpc-json-bridge/build.gradle index c564c66..92101b8 100644 --- a/grpc-json-bridge/build.gradle +++ b/grpc-json-bridge/build.gradle @@ -2,6 +2,8 @@ apply from: rootProject.file('gradle/java.gradle') apply from: rootProject.file('gradle/junit5.gradle') dependencies { + processor 'org.immutables:value' + compile 'io.grpc:grpc-netty' compile 'io.grpc:grpc-services' compile 'org.slf4j:slf4j-api' diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java new file mode 100644 index 0000000..1e1af6a --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java @@ -0,0 +1,30 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.util.JsonFormat; +import org.immutables.value.Value; +import org.immutables.value.Value.Style.ImplementationVisibility; + +@Value.Immutable +@Value.Style(visibility = ImplementationVisibility.PACKAGE) +public interface AvailableMethod { + + /** The {@link Descriptors.MethodDescriptor} for this method. */ + Descriptors.MethodDescriptor methodDescriptor(); + + /** + * A {@link JsonFormat.TypeRegistry} that can be used to serialize and deserialize JSON to communicate with this + * method. + */ + JsonFormat.TypeRegistry typeRegistry(); + + final class Builder extends ImmutableAvailableMethod.Builder {} + + static Builder builder() { + return new Builder(); + } +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java deleted file mode 100644 index 6111cc5..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ChannelProvider.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import io.grpc.Channel; - -public interface ChannelProvider { - - Channel get(String serviceName); - -} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java index d0dc7e6..e8d1c89 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridge.java @@ -4,16 +4,25 @@ package com.github.jared2501.grpc.bridge; +import javax.annotation.Nullable; + public interface GrpcBridge { interface InvocationHandle { void start(); void cancel(); } + enum InvocationErrorType { + SERVICE_NOT_FOUND, + SERVICE_UNAVAILABLE, + METHOD_NOT_FOUND, + NON_UNARY_RESPONSE, + UNKNOWN + } + interface InvocationObserver { + void onError(InvocationErrorType type, @Nullable Throwable error); void onResult(String jsonOutput); - void onMethodNotFound(); - void onError(Throwable error); } /** diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index f4a32ce..b2b7db2 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -4,114 +4,198 @@ package com.github.jared2501.grpc.bridge; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; -import com.google.protobuf.Descriptors; +import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.CallOptions; import io.grpc.Channel; import io.grpc.ClientCall; +import io.grpc.Context; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.Status; +import io.grpc.reflection.v1alpha.FileDescriptorResponse; import io.grpc.reflection.v1alpha.ServerReflectionGrpc; +import io.grpc.reflection.v1alpha.ServerReflectionRequest; +import io.grpc.reflection.v1alpha.ServerReflectionResponse; +import io.grpc.reflection.v1alpha.ServiceResponse; +import io.grpc.stub.StreamObserver; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; final class GrpcBridgeImpl implements GrpcBridge { - private final ChannelProvider channelProvider; + private static final Logger log = LoggerFactory.getLogger(GrpcBridgeImpl.class); - public GrpcBridgeImpl(ChannelProvider channelProvider) { - this.channelProvider = channelProvider; + private final ServiceIndex.Provider serviceIndexFactory; + private final Map serviceChannels = Maps.newConcurrentMap(); + private final Map serviceIndices = Maps.newConcurrentMap(); + + GrpcBridgeImpl(ServiceIndex.Provider serviceIndexFactory) { + this.serviceIndexFactory = serviceIndexFactory; } @Override public InvocationHandle invoke( - String serviceName, String fullMethodName, String jsonInput, InvocationObserver observer) { + String serverName, String fullMethodName, String jsonInput, InvocationObserver observer) { return new InvocationHandle() { + + private final Object lock = new Object(); + private boolean cancelled = false; + private ClientCall call; + @Override public void start() { - Channel channel = channelProvider.get(serviceName); - ReflectionResponseObserver reflection = new ReflectionResponseObserver(serviceName); - reflection.onDoneHandler(() -> { - Optional maybeMethod = reflection.getAvailableMethod(fullMethodName); - if (!maybeMethod.isPresent()) { - observer.onMethodNotFound(); + Channel channel = serviceChannels.get(serverName); + ServiceIndex serviceIndex = serviceIndices.get(serverName); + + if (channel == null) { + observer.onError(InvocationErrorType.SERVICE_NOT_FOUND, null); + return; + } else if (serviceIndex == null) { + observer.onError(InvocationErrorType.SERVICE_UNAVAILABLE, null); + return; + } + + Optional maybeMethod = serviceIndex.getMethod(fullMethodName); + if (!maybeMethod.isPresent()) { + observer.onError(InvocationErrorType.METHOD_NOT_FOUND, null); + return; + } + AvailableMethod method = maybeMethod.get(); + + // Generate the request + DynamicMessage.Builder request = DynamicMessage.newBuilder(method.methodDescriptor().getInputType()); + JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(method.typeRegistry()); + try { + parser.merge(jsonInput, request); + } catch (InvalidProtocolBufferException e) { + observer.onError( + InvocationErrorType.UNKNOWN, + new RuntimeException("Exception encountered when converting JSON to proto", e)); + return; + } + + // Start the call + synchronized (lock) { + if (cancelled) { return; } - Descriptors.MethodDescriptor method = maybeMethod.get(); - - ClientCall call = channel.newCall( - getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); + call = channel.newCall(getMethodDescriptor(fullMethodName), CallOptions.DEFAULT); call.start( new ClientCall.Listener() { + + private byte[] firstMessage; + @Override - public void onHeaders(Metadata headers) { - System.out.println("headers: " + headers); - } + public void onHeaders(Metadata headers) {} @Override public void onMessage(byte[] messageBytes) { + if (firstMessage == null) { + firstMessage = messageBytes; + } else { + observer.onError(InvocationErrorType.NON_UNARY_RESPONSE, null); + call.halfClose(); + call.cancel("Expected unary response", new RuntimeException()); + } + } + + @Override + public void onClose(Status status, Metadata trailers) { + if (status == Status.OK) { + deliverMessage(); + } else { + // TODO: deliver error + } + } + + @Override + public void onReady() {} + + private void deliverMessage() { DynamicMessage message; try { - message = DynamicMessage.parseFrom(method.getOutputType(), messageBytes); + message = DynamicMessage.parseFrom( + method.methodDescriptor().getOutputType(), firstMessage); } catch (InvalidProtocolBufferException e) { - observer.onError(new RuntimeException( - "InvalidProtocolBufferException encountered when parsing message", e)); + observer.onError( + InvocationErrorType.UNKNOWN, + new RuntimeException("Exception encountered when parsing message", e)); return; } + StringBuilder jsonOutput = new StringBuilder(); try { JsonFormat.printer() - .usingTypeRegistry(reflection.getTypeRegistry()) - .appendTo(message, System.out); + .usingTypeRegistry(method.typeRegistry()) + .appendTo(message, jsonOutput); } catch (IOException e) { - observer.onError(new RuntimeException( - "IOException encountered when printing response", e)); + observer.onError( + InvocationErrorType.UNKNOWN, + new RuntimeException( + "Exception encountered when printing response", e)); + return; } - } - - @Override - public void onClose(Status status, Metadata trailers) { - System.out.println("closed! " + status); - reflection.cancel(); - } - @Override - public void onReady() { + observer.onResult(jsonOutput.toString()); } }, new Metadata()); - - DynamicMessage.Builder message = DynamicMessage.newBuilder(method.getInputType()); - JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(reflection.getTypeRegistry()); - try { - parser.merge(jsonInput, message); - } catch (InvalidProtocolBufferException e) { - observer.onError(new RuntimeException( - "InvalidProtocolBufferException encountered when converting JSON to proto", e)); - return; - } - - call.sendMessage(message.build().toByteArray()); + call.sendMessage(request.build().toByteArray()); call.halfClose(); - call.request(2); - }); - reflection.start(ServerReflectionGrpc.newStub(channel)); + call.request(2); // Request two messages to detect if method is unary or not + } } @Override public void cancel() { - // TODO(jnewman): implement! + synchronized (lock) { + cancelled = true; + if (call != null) { + call.halfClose(); + call.cancel("Cancellation requested", new RuntimeException()); + call = null; + } + } } }; } - private MethodDescriptor getMethodDescriptor(String fullMethodName) { + private void updateServices(Map newServices) { + Iterables.removeIf(serviceChannels.keySet(), serverName -> !newServices.containsKey(serverName)); + Iterables.removeIf(serviceIndices.keySet(), serverName -> !newServices.containsKey(serverName)); + + serviceChannels.putAll(newServices); + for (Map.Entry newServer : newServices.entrySet()) { + if (!serviceIndices.containsKey(newServer.getKey())) { + ServiceIndex index = serviceIndexFactory.create(); + serviceIndices.put(newServer.getKey(), index); + updateService(newServer.getKey(), newServer.getValue(), index); + } + } + } + + private void updateService(String serverName, Channel channel, ServiceIndex index) { + // TODO handle failure case + ReflectionResponseObserver observer = new ReflectionResponseObserver(serverName, index, () -> { + }); + observer.start(ServerReflectionGrpc.newStub(channel)); + } + + private static MethodDescriptor getMethodDescriptor(String fullMethodName) { return MethodDescriptor.newBuilder(ByteMarshaller.INSTANCE, ByteMarshaller.INSTANCE) .setType(MethodDescriptor.MethodType.UNARY) .setFullMethodName(fullMethodName) @@ -138,4 +222,127 @@ public byte[] parse(InputStream stream) { } } + private static class ReflectionResponseObserver implements StreamObserver { + + private final String serviceName; + private final ServiceIndex index; + private final Runnable onErrorHandler; + private final Set outstandingRequests = Sets.newHashSet(); + + private StreamObserver reqStream; + private Context.CancellableContext context; + + ReflectionResponseObserver(String serviceName, ServiceIndex index, Runnable onErrorHandler) { + this.serviceName = serviceName; + this.index = index; + this.onErrorHandler = onErrorHandler; + } + + public void start(ServerReflectionGrpc.ServerReflectionStub serverReflection) { + context = Context.CancellableContext.current().withCancellation(); + context.run(() -> reqStream = serverReflection.serverReflectionInfo(this)); + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setListServices("true") + .build()); + } + + public void cancel() { + context.cancel(new RuntimeException()); + } + + @Override + public void onNext(ServerReflectionResponse response) { + if (context.isCancelled()) { + log.info("Future cancelled, not proceeding..."); + return; + } + + outstandingRequests.remove(response.getOriginalRequest()); + + switch (response.getMessageResponseCase()) { + case LIST_SERVICES_RESPONSE: + requestAllFilesForServices(reqStream, outstandingRequests, response); + break; + case FILE_DESCRIPTOR_RESPONSE: + requestUnseenDependencyProtos( + reqStream, + outstandingRequests, + response.getFileDescriptorResponse()); + break; + default: + log.error("Unexpected response case: {}", response.getMessageResponseCase()); + break; + } + + if (outstandingRequests.isEmpty()) { + index.complete(); + } + } + + @Override + public void onError(Throwable error) { + log.error( + "Throwable encountered when streaming handling reflecting for service {}", + serviceName, error); + onErrorHandler.run(); + } + + @Override + public void onCompleted() { + log.info("Reflection complete, service {} likely shutting down.", serviceName); + onErrorHandler.run(); + } + + private void requestAllFilesForServices( + StreamObserver reqStream, + Set outstandingRequests, + ServerReflectionResponse response) { + for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { + index.addAvailableService(service.getName()); + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileContainingSymbol(service.getName()) + .build()); + } + } + + private void requestUnseenDependencyProtos( + StreamObserver reqStream, + Set outstandingRequests, + FileDescriptorResponse response) { + for (ByteString protoBytes : response.getFileDescriptorProtoList()) { + DescriptorProtos.FileDescriptorProto protoDescriptor; + try { + protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); + } catch (InvalidProtocolBufferException e) { + log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); + continue; + } + + index.addProto(protoDescriptor.getName(), protoDescriptor); + + for (String dependencyFileName : protoDescriptor.getDependencyList()) { + if (!index.containsProto(dependencyFileName)) { + makeRequest( + reqStream, + outstandingRequests, + ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); + } + } + } + } + + private static void makeRequest( + StreamObserver reqStream, + Set outstandingRequests, + ServerReflectionRequest request) { + outstandingRequests.add(request); + reqStream.onNext(request); + } + } + } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java deleted file mode 100644 index 6b41c78..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionResponseObserver.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import static com.google.common.base.Preconditions.checkState; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.collect.TreeTraverser; -import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos.FileDescriptorProto; -import com.google.protobuf.Descriptors.DescriptorValidationException; -import com.google.protobuf.Descriptors.FileDescriptor; -import com.google.protobuf.Descriptors.MethodDescriptor; -import com.google.protobuf.Descriptors.ServiceDescriptor; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import io.grpc.Context; -import io.grpc.reflection.v1alpha.FileDescriptorResponse; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; -import io.grpc.reflection.v1alpha.ServerReflectionRequest; -import io.grpc.reflection.v1alpha.ServerReflectionResponse; -import io.grpc.reflection.v1alpha.ServiceResponse; -import io.grpc.stub.StreamObserver; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class ReflectionResponseObserver implements StreamObserver { - - private static final Logger log = LoggerFactory.getLogger(ReflectionResponseObserver.class); - - private final Set outstandingRequests = Sets.newHashSet(); - private final Map protosByFileName = Maps.newHashMap(); - private final String serviceName; - - private Runnable onDoneHandler; - private StreamObserver reqStream; - private Context.CancellableContext context; - - private final Set availableServices = Sets.newHashSet(); - private final Map availableMethodsByFullMethodName = Maps.newHashMap(); - private JsonFormat.TypeRegistry typeRegistry; - - public ReflectionResponseObserver(String serviceName) { - this.serviceName = serviceName; - } - - public void start(ServerReflectionGrpc.ServerReflectionStub serverReflection) { - context = Context.CancellableContext.current().withCancellation(); - context.run(() -> reqStream = serverReflection.serverReflectionInfo(this)); - reqStream.onNext(ServerReflectionRequest.newBuilder() - .setListServices("true") - .build()); - } - - public void onDoneHandler(Runnable runnable) { - this.onDoneHandler = runnable; - } - - public void cancel() { - context.cancel(new RuntimeException()); - } - - public JsonFormat.TypeRegistry getTypeRegistry() { - checkState(typeRegistry != null, "ReflectionResponseObserver has not finished yet"); - return typeRegistry; - } - - public Optional getAvailableMethod(String fullMethodName) { - checkState(typeRegistry != null, "ReflectionResponseObserver has not finished yet"); - return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); - } - - @Override - public void onNext(ServerReflectionResponse response) { - if (context.isCancelled()) { - log.info("Future cancelled, not proceeding..."); - return; - } - - outstandingRequests.remove(response.getOriginalRequest()); - - switch (response.getMessageResponseCase()) { - case LIST_SERVICES_RESPONSE: - requestAllFilesForServices(reqStream, outstandingRequests, response); - break; - case FILE_DESCRIPTOR_RESPONSE: - requestUnseenDependencyProtos( - reqStream, - outstandingRequests, - protosByFileName, - response.getFileDescriptorResponse()); - break; - default: - log.error("Unexpected response case: {}", response.getMessageResponseCase()); - break; - } - - if (outstandingRequests.isEmpty()) { - JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); - Collection compiledProtos = compileProtos(protosByFileName); - for (FileDescriptor compiledProto : compiledProtos) { - typeRegistryBuilder.add(compiledProto.getMessageTypes()); - for (ServiceDescriptor service : compiledProto.getServices()) { - if (availableServices.contains(service.getFullName())) { - for (MethodDescriptor method : service.getMethods()) { - String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); - availableMethodsByFullMethodName.put(methodFullName, method); - } - } - } - } - typeRegistry = typeRegistryBuilder.build(); - onDoneHandler.run(); - } - } - - @Override - public void onError(Throwable error) { - log.error( - "Throwable encountered when streaming handling reflecting for service {}", - serviceName, error); - // TODO(jnewman): handle errors here better - } - - @Override - public void onCompleted() { - log.info("Reflection complete, service {} likely shutting down.", serviceName); - // TODO(jnewman): handle this better - } - - private void requestAllFilesForServices( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionResponse response) { - for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { - availableServices.add(service.getName()); - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileContainingSymbol(service.getName()) - .build()); - } - } - - private static void requestUnseenDependencyProtos( - StreamObserver reqStream, - Set outstandingRequests, - Map protosByFileName, - FileDescriptorResponse response) { - for (ByteString protoBytes : response.getFileDescriptorProtoList()) { - FileDescriptorProto protoDescriptor; - try { - protoDescriptor = FileDescriptorProto.parseFrom(protoBytes); - } catch (InvalidProtocolBufferException e) { - log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); - continue; - } - - protosByFileName.put(protoDescriptor.getName(), protoDescriptor); - - for (String dependencyFileName : protoDescriptor.getDependencyList()) { - if (!protosByFileName.containsKey(dependencyFileName)) { - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileByFilename(dependencyFileName) - .build()); - } - } - } - } - - private static Collection compileProtos( - Map protosByFileName) { - // Find all "roots", where a root is a proto file for which another proto file does not depend on it - Map rootsByFileName = Maps.newHashMap(protosByFileName); - for (FileDescriptorProto proto : protosByFileName.values()) { - for (String dependencyFileName : proto.getDependencyList()) { - rootsByFileName.remove(dependencyFileName); - } - } - - Map compiledProtosByFileName = Maps.newHashMap(); - - // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto - // file if it has not already been compiled - TreeTraverser treeTraverser = new TreeTraverser() { - @Override - public Iterable children(FileDescriptorProto root) { - return root.getDependencyList() - .stream() - // Note: skip visiting dependencies if they have already been compiled - .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) - .map(protosByFileName::get) - .collect(Collectors.toSet()); - } - }; - for (FileDescriptorProto root : rootsByFileName.values()) { - for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { - FileDescriptor[] dependencies = proto.getDependencyList() - .stream() - .map(compiledProtosByFileName::get) - .toArray(FileDescriptor[]::new); - - FileDescriptor compiledProto; - try { - compiledProto = FileDescriptor.buildFrom(proto, dependencies); - } catch (DescriptorValidationException e) { - log.warn("Exception encountered when building proto... skipping", e); - continue; - } - - compiledProtosByFileName.put(proto.getName(), compiledProto); - } - } - - return compiledProtosByFileName.values(); - } - - private static void makeRequest( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionRequest request) { - outstandingRequests.add(request); - reqStream.onNext(request); - } - -} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java new file mode 100644 index 0000000..fb01375 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java @@ -0,0 +1,30 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.protobuf.DescriptorProtos; +import java.util.Optional; + +public interface ServiceIndex { + + interface Provider { + ServiceIndex create(); + } + + void restart(); + + void addAvailableService(String serviceName); + + void addProto(String fileName, DescriptorProtos.FileDescriptorProto proto); + + void complete(); + + boolean isAvailable(); + + boolean containsProto(String fileName); + + Optional getMethod(String fullMethodName); + +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java new file mode 100644 index 0000000..b7da8f0 --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java @@ -0,0 +1,135 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeTraverser; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.util.JsonFormat; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ServiceIndexImpl implements ServiceIndex { + + private static final Logger log = LoggerFactory.getLogger(ServiceIndexImpl.class); + + private final Set availableServices = Sets.newHashSet(); + private final Map protosByFileName = Maps.newHashMap(); + + private final Map availableMethodsByFullMethodName = Maps.newHashMap(); + private boolean isComplete = false; + + + @Override + public void restart() { + availableServices.clear(); + protosByFileName.clear(); + } + + @Override + public void addAvailableService(String serviceName) { + availableServices.add(serviceName); + } + + @Override + public void addProto(String fileName, FileDescriptorProto proto) { + protosByFileName.put(fileName, proto); + } + + @Override + public void complete() { + isComplete = true; + buildIndex(availableServices, compileProtos(protosByFileName)); + } + + @Override + public boolean isAvailable() { + return isComplete; + } + + @Override + public boolean containsProto(String fileName) { + return protosByFileName.containsKey(fileName); + } + + @Override + public Optional getMethod(String fullMethodName) { + return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); + } + + private static Collection compileProtos(Map protosByFileName) { + // Find all "roots", where a root is a proto file for which another proto file does not depend on it + Map rootsByFileName = Maps.newHashMap(protosByFileName); + for (FileDescriptorProto proto : protosByFileName.values()) { + for (String dependencyFileName : proto.getDependencyList()) { + rootsByFileName.remove(dependencyFileName); + } + } + + Map compiledProtosByFileName = Maps.newHashMap(); + + // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto + // file if it has not already been compiled + TreeTraverser treeTraverser = new TreeTraverser() { + @Override + public Iterable children(FileDescriptorProto root) { + return root.getDependencyList() + .stream() + // Note: skip visiting dependencies if they have already been compiled + .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) + .map(protosByFileName::get) + .collect(Collectors.toSet()); + } + }; + for (FileDescriptorProto root : rootsByFileName.values()) { + for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + FileDescriptor[] dependencies = proto.getDependencyList() + .stream() + .map(compiledProtosByFileName::get) + .toArray(FileDescriptor[]::new); + + FileDescriptor compiledProto; + try { + compiledProto = FileDescriptor.buildFrom(proto, dependencies); + } catch (DescriptorValidationException e) { + log.warn("Exception encountered when building proto... skipping", e); + continue; + } + + compiledProtosByFileName.put(proto.getName(), compiledProto); + } + } + + return compiledProtosByFileName.values(); + } + + private void buildIndex( + Set availableServices, + Collection compiledProtos) { + JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); + for (FileDescriptor compiledProto : compiledProtos) { + typeRegistryBuilder.add(compiledProto.getMessageTypes()); + for (Descriptors.ServiceDescriptor service : compiledProto.getServices()) { + if (availableServices.contains(service.getFullName())) { + for (Descriptors.MethodDescriptor method : service.getMethods()) { + String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); + availableMethodsByFullMethodName.put(methodFullName, method); + } + } + } + } + typeRegistry = typeRegistryBuilder.build(); + } + +} diff --git a/versions.props b/versions.props index f60da18..7ab67a3 100644 --- a/versions.props +++ b/versions.props @@ -4,6 +4,7 @@ io.dropwizard:* = 1.2.3 io.grpc:* = 1.7.1 org.apiguardian:apiguardian-api = 1.0.0 org.assertj:assertj-core = 3.6.2 +org.immutables:* = 2.5.6 org.junit.jupiter:* = 5.0.2 org.mockito:* = 2.13.0 org.slf4j:* = 1.7.25 From fda4cea1a9040f1f40724af598762198b87845ba Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Fri, 6 Apr 2018 13:08:14 -0700 Subject: [PATCH 09/11] wip --- grpc-json-bridge/build.gradle | 1 + .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 137 ------------------ .../grpc/bridge/ReflectionChannel.java | 40 +++++ .../grpc/bridge/ReflectionChannelImpl.java | 131 +++++++++++++++++ .../jared2501/grpc/bridge/ServiceIndex.java | 13 -- .../grpc/bridge/ServiceIndexImpl.java | 12 +- .../bridge/GrpcBridgeImplIntegrationTest.java | 52 +++---- .../bridge/ReflectionChannelImplTest.java | 77 ++++++++++ .../grpc/bridge/TestReflectionObserver.java | 57 ++++++++ 9 files changed, 336 insertions(+), 184 deletions(-) create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java create mode 100644 grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java create mode 100644 grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/TestReflectionObserver.java diff --git a/grpc-json-bridge/build.gradle b/grpc-json-bridge/build.gradle index 92101b8..d60ad40 100644 --- a/grpc-json-bridge/build.gradle +++ b/grpc-json-bridge/build.gradle @@ -11,5 +11,6 @@ dependencies { testCompile project(':test-grpc-service') testCompile 'org.assertj:assertj-core' + testCompile 'org.mockito:mockito-core' testCompile 'org.slf4j:slf4j-simple' } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index b2b7db2..03c1b9c 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -6,32 +6,21 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; -import com.google.protobuf.ByteString; -import com.google.protobuf.DescriptorProtos; import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.grpc.CallOptions; import io.grpc.Channel; import io.grpc.ClientCall; -import io.grpc.Context; import io.grpc.Metadata; import io.grpc.MethodDescriptor; import io.grpc.Status; -import io.grpc.reflection.v1alpha.FileDescriptorResponse; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; -import io.grpc.reflection.v1alpha.ServerReflectionRequest; -import io.grpc.reflection.v1alpha.ServerReflectionResponse; -import io.grpc.reflection.v1alpha.ServiceResponse; -import io.grpc.stub.StreamObserver; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Map; import java.util.Optional; -import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -190,9 +179,6 @@ private void updateServices(Map newServices) { private void updateService(String serverName, Channel channel, ServiceIndex index) { // TODO handle failure case - ReflectionResponseObserver observer = new ReflectionResponseObserver(serverName, index, () -> { - }); - observer.start(ServerReflectionGrpc.newStub(channel)); } private static MethodDescriptor getMethodDescriptor(String fullMethodName) { @@ -222,127 +208,4 @@ public byte[] parse(InputStream stream) { } } - private static class ReflectionResponseObserver implements StreamObserver { - - private final String serviceName; - private final ServiceIndex index; - private final Runnable onErrorHandler; - private final Set outstandingRequests = Sets.newHashSet(); - - private StreamObserver reqStream; - private Context.CancellableContext context; - - ReflectionResponseObserver(String serviceName, ServiceIndex index, Runnable onErrorHandler) { - this.serviceName = serviceName; - this.index = index; - this.onErrorHandler = onErrorHandler; - } - - public void start(ServerReflectionGrpc.ServerReflectionStub serverReflection) { - context = Context.CancellableContext.current().withCancellation(); - context.run(() -> reqStream = serverReflection.serverReflectionInfo(this)); - reqStream.onNext(ServerReflectionRequest.newBuilder() - .setListServices("true") - .build()); - } - - public void cancel() { - context.cancel(new RuntimeException()); - } - - @Override - public void onNext(ServerReflectionResponse response) { - if (context.isCancelled()) { - log.info("Future cancelled, not proceeding..."); - return; - } - - outstandingRequests.remove(response.getOriginalRequest()); - - switch (response.getMessageResponseCase()) { - case LIST_SERVICES_RESPONSE: - requestAllFilesForServices(reqStream, outstandingRequests, response); - break; - case FILE_DESCRIPTOR_RESPONSE: - requestUnseenDependencyProtos( - reqStream, - outstandingRequests, - response.getFileDescriptorResponse()); - break; - default: - log.error("Unexpected response case: {}", response.getMessageResponseCase()); - break; - } - - if (outstandingRequests.isEmpty()) { - index.complete(); - } - } - - @Override - public void onError(Throwable error) { - log.error( - "Throwable encountered when streaming handling reflecting for service {}", - serviceName, error); - onErrorHandler.run(); - } - - @Override - public void onCompleted() { - log.info("Reflection complete, service {} likely shutting down.", serviceName); - onErrorHandler.run(); - } - - private void requestAllFilesForServices( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionResponse response) { - for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { - index.addAvailableService(service.getName()); - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileContainingSymbol(service.getName()) - .build()); - } - } - - private void requestUnseenDependencyProtos( - StreamObserver reqStream, - Set outstandingRequests, - FileDescriptorResponse response) { - for (ByteString protoBytes : response.getFileDescriptorProtoList()) { - DescriptorProtos.FileDescriptorProto protoDescriptor; - try { - protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); - } catch (InvalidProtocolBufferException e) { - log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); - continue; - } - - index.addProto(protoDescriptor.getName(), protoDescriptor); - - for (String dependencyFileName : protoDescriptor.getDependencyList()) { - if (!index.containsProto(dependencyFileName)) { - makeRequest( - reqStream, - outstandingRequests, - ServerReflectionRequest.newBuilder() - .setFileByFilename(dependencyFileName) - .build()); - } - } - } - } - - private static void makeRequest( - StreamObserver reqStream, - Set outstandingRequests, - ServerReflectionRequest request) { - outstandingRequests.add(request); - reqStream.onNext(request); - } - } - } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java new file mode 100644 index 0000000..12c9eee --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java @@ -0,0 +1,40 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.protobuf.DescriptorProtos; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc.ServerReflectionStub; + +public interface ReflectionChannel { + interface ReflectionObserver { + /** Invoked for every available gRPC service. */ + void onAvailableService(String serviceName); + + /** Invoked for every proto file. */ + void onProtoFile(String fileName, DescriptorProtos.FileDescriptorProto proto); + + /** + * Invoked once all services and proto files have been discovered. No further methods will be invoked on the + * observer if this method is invoked. + */ + void onComplete(); + + /** + * Invoked if there are any expected errors.No further methods will be invoked on the observer if this method + * is invoked. + */ + void onError(Throwable error); + } + + interface ReflectionCall { + /** + * Cancels the reflection call with the given error. If the call is not complete then the + * {@link ReflectionObserver} will be notified via {@link ReflectionObserver#onError}. + */ + void cancel(Throwable reason); + } + + ReflectionCall startCall(String serviceName, ServerReflectionStub serverReflection, ReflectionObserver observer); +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java new file mode 100644 index 0000000..5ea74da --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java @@ -0,0 +1,131 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.collect.Sets; +import com.google.protobuf.ByteString; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.InvalidProtocolBufferException; +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.reflection.v1alpha.FileDescriptorResponse; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc.ServerReflectionStub; +import io.grpc.reflection.v1alpha.ServerReflectionRequest; +import io.grpc.reflection.v1alpha.ServerReflectionResponse; +import io.grpc.reflection.v1alpha.ServiceResponse; +import io.grpc.stub.StreamObserver; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class ReflectionChannelImpl implements ReflectionChannel { + + private static final Logger log = LoggerFactory.getLogger(ReflectionChannelImpl.class); + + @Override + public ReflectionCall startCall( + String serviceName, ServerReflectionStub serverReflection, ReflectionObserver observer) { + Context.CancellableContext context = Context.CancellableContext.current().withCancellation(); + context.run(() -> { + ReflectionResponseObserver streamObs = new ReflectionResponseObserver(serviceName, observer); + StreamObserver reqStream = serverReflection.serverReflectionInfo(streamObs); + streamObs.start(reqStream); + }); + return context::cancel; + } + + private static class ReflectionResponseObserver implements StreamObserver { + + private final String serviceName; + private final ReflectionObserver observer; + private final Set observedProtoFiles = Sets.newHashSet(); + private final Set outstandingRequests = Sets.newHashSet(); + + private StreamObserver reqStream; + + ReflectionResponseObserver(String serviceName, ReflectionObserver observer) { + this.serviceName = serviceName; + this.observer = observer; + } + + public void start(StreamObserver requestStreamToRun) { + reqStream = requestStreamToRun; + reqStream.onNext(ServerReflectionRequest.newBuilder() + .setListServices("true") + .build()); + } + + @Override + public void onNext(ServerReflectionResponse response) { + outstandingRequests.remove(response.getOriginalRequest()); + + switch (response.getMessageResponseCase()) { + case LIST_SERVICES_RESPONSE: + requestAllFilesForServices(response); + break; + case FILE_DESCRIPTOR_RESPONSE: + requestUnseenDependencyProtos(response.getFileDescriptorResponse()); + break; + default: + log.error("Unexpected response case: {}", response.getMessageResponseCase()); + break; + } + + if (outstandingRequests.isEmpty()) { + reqStream.onCompleted(); + observer.onComplete(); + } + } + + @Override + public void onError(Throwable error) { + log.debug("Throwable encountered when streaming handling reflecting for service {}", serviceName, error); + observer.onError(error); + } + + @Override + public void onCompleted() { + log.debug("Reflection complete, service {} likely shutting down.", serviceName); + observer.onError(Status.UNAVAILABLE.asException()); + } + + private void requestAllFilesForServices(ServerReflectionResponse response) { + for (ServiceResponse service : response.getListServicesResponse().getServiceList()) { + observer.onAvailableService(service.getName()); + makeRequest(ServerReflectionRequest.newBuilder() + .setFileContainingSymbol(service.getName()) + .build()); + } + } + + private void requestUnseenDependencyProtos(FileDescriptorResponse response) { + for (ByteString protoBytes : response.getFileDescriptorProtoList()) { + DescriptorProtos.FileDescriptorProto protoDescriptor; + try { + protoDescriptor = DescriptorProtos.FileDescriptorProto.parseFrom(protoBytes); + } catch (InvalidProtocolBufferException e) { + log.warn("InvalidProtocolBufferException when parsing proto bytes... skipping", e); + continue; + } + + observer.onProtoFile(protoDescriptor.getName(), protoDescriptor); + observedProtoFiles.add(protoDescriptor.getName()); + + for (String dependencyFileName : protoDescriptor.getDependencyList()) { + if (!observedProtoFiles.contains(dependencyFileName)) { + makeRequest(ServerReflectionRequest.newBuilder() + .setFileByFilename(dependencyFileName) + .build()); + } + } + } + } + + private void makeRequest(ServerReflectionRequest request) { + outstandingRequests.add(request); + reqStream.onNext(request); + } + } +} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java index fb01375..6fb8070 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java @@ -4,7 +4,6 @@ package com.github.jared2501.grpc.bridge; -import com.google.protobuf.DescriptorProtos; import java.util.Optional; public interface ServiceIndex { @@ -13,18 +12,6 @@ interface Provider { ServiceIndex create(); } - void restart(); - - void addAvailableService(String serviceName); - - void addProto(String fileName, DescriptorProtos.FileDescriptorProto proto); - - void complete(); - - boolean isAvailable(); - - boolean containsProto(String fileName); - Optional getMethod(String fullMethodName); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java index b7da8f0..4fe893f 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java @@ -31,34 +31,28 @@ class ServiceIndexImpl implements ServiceIndex { private boolean isComplete = false; - @Override public void restart() { availableServices.clear(); protosByFileName.clear(); } - @Override public void addAvailableService(String serviceName) { availableServices.add(serviceName); } - @Override public void addProto(String fileName, FileDescriptorProto proto) { protosByFileName.put(fileName, proto); } - @Override public void complete() { isComplete = true; buildIndex(availableServices, compileProtos(protosByFileName)); } - @Override public boolean isAvailable() { return isComplete; } - @Override public boolean containsProto(String fileName) { return protosByFileName.containsKey(fileName); } @@ -124,12 +118,14 @@ private void buildIndex( if (availableServices.contains(service.getFullName())) { for (Descriptors.MethodDescriptor method : service.getMethods()) { String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); - availableMethodsByFullMethodName.put(methodFullName, method); + availableMethodsByFullMethodName.put(methodFullName, AvailableMethod.builder() + .methodDescriptor(method) + .typeRegistry(null) // TODO this?? + .build()); } } } } - typeRegistry = typeRegistryBuilder.build(); } } diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java index 581d13d..e154edf 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/GrpcBridgeImplIntegrationTest.java @@ -42,31 +42,31 @@ void tearDown() throws InterruptedException { @Test void norfquix() throws ExecutionException, InterruptedException { - GrpcBridge bridge = new GrpcBridgeImpl(serviceName -> channel); - - GrpcBridge.InvocationHandle handle = bridge.invoke( - "foo", - "com.github.jared2501.grpc.bridge.test.TestService/UnaryReqUnaryResp", - "{\"message\": \"message\", \"empty\": {}}", - new GrpcBridge.InvocationObserver() { - @Override - public void onResult(String jsonOutput) { - System.out.println("output: " + jsonOutput); - } - - @Override - public void onMethodNotFound() { - System.out.println("method not found!"); - } - - @Override - public void onError(Throwable error) { - error.printStackTrace(); - } - }); - - handle.start(); - - Thread.sleep(5000); + // GrpcBridge bridge = new GrpcBridgeImpl(serviceName -> channel); + // + // GrpcBridge.InvocationHandle handle = bridge.invoke( + // "foo", + // "com.github.jared2501.grpc.bridge.test.TestService/UnaryReqUnaryResp", + // "{\"message\": \"message\", \"empty\": {}}", + // new GrpcBridge.InvocationObserver() { + // @Override + // public void onResult(String jsonOutput) { + // System.out.println("output: " + jsonOutput); + // } + // + // @Override + // public void onMethodNotFound() { + // System.out.println("method not found!"); + // } + // + // @Override + // public void onError(Throwable error) { + // error.printStackTrace(); + // } + // }); + // + // handle.start(); + // + // Thread.sleep(5000); } } diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java new file mode 100644 index 0000000..bfd63c6 --- /dev/null +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java @@ -0,0 +1,77 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.jared2501.grpc.bridge.test.TestServiceImpl; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ReflectionChannelImplTest { + + private Server server; + private ManagedChannel channel; + private ReflectionChannel reflection; + + @BeforeEach + void setUp() throws IOException { + server = InProcessServerBuilder.forName("test") + .addService(TestServiceImpl.INSTANCE) + .addService(ProtoReflectionService.newInstance()) + .build() + .start(); + channel = InProcessChannelBuilder.forName("test").build(); + reflection = new ReflectionChannelImpl(); + } + + @AfterEach + void tearDown() throws InterruptedException { + channel.shutdown(); + channel.awaitTermination(5, TimeUnit.HOURS); + server.shutdown(); + server.awaitTermination(); + } + + @Test + void successfulRoundTrip() { + TestReflectionObserver observer = new TestReflectionObserver(); + reflection.startCall("test", ServerReflectionGrpc.newStub(channel), observer); + + observer.waitUntilComplete(); + assertThat(observer.getError()).isNull(); + assertThat(observer.getAvailableService()).containsExactly( + "com.github.jared2501.grpc.bridge.test.TestService", + "grpc.reflection.v1alpha.ServerReflection"); + assertThat(observer.getProtoFiles().keySet()).containsExactly( + "test.proto", + "google/protobuf/empty.proto", + "transitive.proto", + "io/grpc/reflection/v1alpha/reflection.proto"); + } + + @Test + void unavailableChannel() throws InterruptedException { + channel.shutdown(); + channel.awaitTermination(5, TimeUnit.HOURS); + TestReflectionObserver observer = new TestReflectionObserver(); + reflection.startCall("test", ServerReflectionGrpc.newStub(channel), observer); + observer.waitUntilComplete(); + assertThat(observer.getError()) + .isInstanceOf(StatusRuntimeException.class) + .matches(error -> ((StatusRuntimeException) error).getStatus().getCode() == Status.Code.UNAVAILABLE); + } +} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/TestReflectionObserver.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/TestReflectionObserver.java new file mode 100644 index 0000000..e0aa8f0 --- /dev/null +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/TestReflectionObserver.java @@ -0,0 +1,57 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.collect.Maps; +import com.google.protobuf.DescriptorProtos; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Semaphore; +import org.assertj.core.util.Sets; + +public final class TestReflectionObserver implements ReflectionChannel.ReflectionObserver { + + private Set availableService = Sets.newHashSet(); + private Map protoFiles = Maps.newHashMap(); + private Throwable error; + private final Semaphore done = new Semaphore(0); + + @Override + public void onAvailableService(String serviceName) { + availableService.add(serviceName); + } + + @Override + public void onProtoFile(String fileName, DescriptorProtos.FileDescriptorProto proto) { + protoFiles.put(fileName, proto); + } + + @Override + public void onComplete() { + done.release(1); + } + + @Override + public void onError(Throwable err) { + error = err; + done.release(1); + } + + public void waitUntilComplete() { + done.acquireUninterruptibly(1); + } + + public Throwable getError() { + return error; + } + + public Set getAvailableService() { + return availableService; + } + + public Map getProtoFiles() { + return protoFiles; + } +} From c8530db9b42dbdd3867a3d6ac983d4fa587d8998 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Fri, 6 Apr 2018 13:15:15 -0700 Subject: [PATCH 10/11] wip --- .../grpc/bridge/AvailableMethod.java | 30 ---- .../jared2501/grpc/bridge/GrpcBridgeImpl.java | 11 +- .../grpc/bridge/ReflectionChannel.java | 5 +- .../grpc/bridge/ReflectionChannelImpl.java | 6 +- .../jared2501/grpc/bridge/ServiceIndex.java | 7 +- .../grpc/bridge/ServiceIndexImpl.java | 123 ++--------------- .../grpc/bridge/ServiceIndexImpl2.java | 130 ++++++++++++++++++ 7 files changed, 153 insertions(+), 159 deletions(-) delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java create mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java deleted file mode 100644 index 1e1af6a..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/AvailableMethod.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import com.google.protobuf.Descriptors; -import com.google.protobuf.util.JsonFormat; -import org.immutables.value.Value; -import org.immutables.value.Value.Style.ImplementationVisibility; - -@Value.Immutable -@Value.Style(visibility = ImplementationVisibility.PACKAGE) -public interface AvailableMethod { - - /** The {@link Descriptors.MethodDescriptor} for this method. */ - Descriptors.MethodDescriptor methodDescriptor(); - - /** - * A {@link JsonFormat.TypeRegistry} that can be used to serialize and deserialize JSON to communicate with this - * method. - */ - JsonFormat.TypeRegistry typeRegistry(); - - final class Builder extends ImmutableAvailableMethod.Builder {} - - static Builder builder() { - return new Builder(); - } -} diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java index 03c1b9c..b8016c3 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/GrpcBridgeImpl.java @@ -28,14 +28,9 @@ final class GrpcBridgeImpl implements GrpcBridge { private static final Logger log = LoggerFactory.getLogger(GrpcBridgeImpl.class); - private final ServiceIndex.Provider serviceIndexFactory; private final Map serviceChannels = Maps.newConcurrentMap(); private final Map serviceIndices = Maps.newConcurrentMap(); - GrpcBridgeImpl(ServiceIndex.Provider serviceIndexFactory) { - this.serviceIndexFactory = serviceIndexFactory; - } - @Override public InvocationHandle invoke( String serverName, String fullMethodName, String jsonInput, InvocationObserver observer) { @@ -170,9 +165,9 @@ private void updateServices(Map newServices) { serviceChannels.putAll(newServices); for (Map.Entry newServer : newServices.entrySet()) { if (!serviceIndices.containsKey(newServer.getKey())) { - ServiceIndex index = serviceIndexFactory.create(); - serviceIndices.put(newServer.getKey(), index); - updateService(newServer.getKey(), newServer.getValue(), index); + // ServiceIndex index = serviceIndexFactory.create(); + // serviceIndices.put(newServer.getKey(), index); + // updateService(newServer.getKey(), newServer.getValue(), index); } } } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java index 12c9eee..7d77a3b 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java @@ -5,7 +5,7 @@ package com.github.jared2501.grpc.bridge; import com.google.protobuf.DescriptorProtos; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc.ServerReflectionStub; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; public interface ReflectionChannel { interface ReflectionObserver { @@ -36,5 +36,6 @@ interface ReflectionCall { void cancel(Throwable reason); } - ReflectionCall startCall(String serviceName, ServerReflectionStub serverReflection, ReflectionObserver observer); + ReflectionCall startCall( + String serviceName, ServerReflectionGrpc.ServerReflectionStub reflectionStub, ReflectionObserver observer); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java index 5ea74da..22b8a1a 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java @@ -11,7 +11,7 @@ import io.grpc.Context; import io.grpc.Status; import io.grpc.reflection.v1alpha.FileDescriptorResponse; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc.ServerReflectionStub; +import io.grpc.reflection.v1alpha.ServerReflectionGrpc; import io.grpc.reflection.v1alpha.ServerReflectionRequest; import io.grpc.reflection.v1alpha.ServerReflectionResponse; import io.grpc.reflection.v1alpha.ServiceResponse; @@ -26,11 +26,11 @@ public final class ReflectionChannelImpl implements ReflectionChannel { @Override public ReflectionCall startCall( - String serviceName, ServerReflectionStub serverReflection, ReflectionObserver observer) { + String serviceName, ServerReflectionGrpc.ServerReflectionStub reflectionStub, ReflectionObserver observer) { Context.CancellableContext context = Context.CancellableContext.current().withCancellation(); context.run(() -> { ReflectionResponseObserver streamObs = new ReflectionResponseObserver(serviceName, observer); - StreamObserver reqStream = serverReflection.serverReflectionInfo(streamObs); + StreamObserver reqStream = reflectionStub.serverReflectionInfo(streamObs); streamObs.start(reqStream); }); return context::cancel; diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java index 6fb8070..6d123dd 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndex.java @@ -4,14 +4,11 @@ package com.github.jared2501.grpc.bridge; +import com.google.protobuf.Descriptors; import java.util.Optional; public interface ServiceIndex { - interface Provider { - ServiceIndex create(); - } - - Optional getMethod(String fullMethodName); + Optional getMethod(String fullMethodName); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java index 4fe893f..ff77cd5 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java @@ -4,128 +4,29 @@ package com.github.jared2501.grpc.bridge; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.collect.TreeTraverser; -import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.common.util.concurrent.AbstractService; import com.google.protobuf.Descriptors; -import com.google.protobuf.Descriptors.DescriptorValidationException; -import com.google.protobuf.Descriptors.FileDescriptor; -import com.google.protobuf.util.JsonFormat; -import java.util.Collection; -import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -class ServiceIndexImpl implements ServiceIndex { +public final class ServiceIndexImpl extends AbstractService implements ServiceIndex { - private static final Logger log = LoggerFactory.getLogger(ServiceIndexImpl.class); + private final ReflectionChannel reflectionChannel; - private final Set availableServices = Sets.newHashSet(); - private final Map protosByFileName = Maps.newHashMap(); - - private final Map availableMethodsByFullMethodName = Maps.newHashMap(); - private boolean isComplete = false; - - - public void restart() { - availableServices.clear(); - protosByFileName.clear(); - } - - public void addAvailableService(String serviceName) { - availableServices.add(serviceName); - } - - public void addProto(String fileName, FileDescriptorProto proto) { - protosByFileName.put(fileName, proto); - } - - public void complete() { - isComplete = true; - buildIndex(availableServices, compileProtos(protosByFileName)); - } - - public boolean isAvailable() { - return isComplete; - } - - public boolean containsProto(String fileName) { - return protosByFileName.containsKey(fileName); + public ServiceIndexImpl() { } @Override - public Optional getMethod(String fullMethodName) { - return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); + protected void doStart() { + notifyStarted(); } - private static Collection compileProtos(Map protosByFileName) { - // Find all "roots", where a root is a proto file for which another proto file does not depend on it - Map rootsByFileName = Maps.newHashMap(protosByFileName); - for (FileDescriptorProto proto : protosByFileName.values()) { - for (String dependencyFileName : proto.getDependencyList()) { - rootsByFileName.remove(dependencyFileName); - } - } - - Map compiledProtosByFileName = Maps.newHashMap(); - - // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto - // file if it has not already been compiled - TreeTraverser treeTraverser = new TreeTraverser() { - @Override - public Iterable children(FileDescriptorProto root) { - return root.getDependencyList() - .stream() - // Note: skip visiting dependencies if they have already been compiled - .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) - .map(protosByFileName::get) - .collect(Collectors.toSet()); - } - }; - for (FileDescriptorProto root : rootsByFileName.values()) { - for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { - FileDescriptor[] dependencies = proto.getDependencyList() - .stream() - .map(compiledProtosByFileName::get) - .toArray(FileDescriptor[]::new); - - FileDescriptor compiledProto; - try { - compiledProto = FileDescriptor.buildFrom(proto, dependencies); - } catch (DescriptorValidationException e) { - log.warn("Exception encountered when building proto... skipping", e); - continue; - } - - compiledProtosByFileName.put(proto.getName(), compiledProto); - } - } - - return compiledProtosByFileName.values(); + @Override + protected void doStop() { + notifyStopped(); } - private void buildIndex( - Set availableServices, - Collection compiledProtos) { - JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); - for (FileDescriptor compiledProto : compiledProtos) { - typeRegistryBuilder.add(compiledProto.getMessageTypes()); - for (Descriptors.ServiceDescriptor service : compiledProto.getServices()) { - if (availableServices.contains(service.getFullName())) { - for (Descriptors.MethodDescriptor method : service.getMethods()) { - String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); - availableMethodsByFullMethodName.put(methodFullName, AvailableMethod.builder() - .methodDescriptor(method) - .typeRegistry(null) // TODO this?? - .build()); - } - } - } - } + @Override + public Optional getMethod(String fullMethodName) { + return Optional.empty(); } - } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java new file mode 100644 index 0000000..00f53de --- /dev/null +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java @@ -0,0 +1,130 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeTraverser; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.util.JsonFormat; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ServiceIndexImpl2 implements ServiceIndex { + + private static final Logger log = LoggerFactory.getLogger(ServiceIndexImpl2.class); + + private final Set availableServices = Sets.newHashSet(); + private final Map protosByFileName = Maps.newHashMap(); + + private final Map availableMethodsByFullMethodName = Maps.newHashMap(); + private boolean isComplete = false; + + public void restart() { + availableServices.clear(); + protosByFileName.clear(); + } + + public void addAvailableService(String serviceName) { + availableServices.add(serviceName); + } + + public void addProto(String fileName, FileDescriptorProto proto) { + protosByFileName.put(fileName, proto); + } + + public void complete() { + isComplete = true; + buildIndex(availableServices, compileProtos(protosByFileName)); + } + + public boolean isAvailable() { + return isComplete; + } + + public boolean containsProto(String fileName) { + return protosByFileName.containsKey(fileName); + } + + @Override + public Optional getMethod(String fullMethodName) { + return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); + } + + private static Collection compileProtos(Map protosByFileName) { + // Find all "roots", where a root is a proto file for which another proto file does not depend on it + Map rootsByFileName = Maps.newHashMap(protosByFileName); + for (FileDescriptorProto proto : protosByFileName.values()) { + for (String dependencyFileName : proto.getDependencyList()) { + rootsByFileName.remove(dependencyFileName); + } + } + + Map compiledProtosByFileName = Maps.newHashMap(); + + // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto + // file if it has not already been compiled + TreeTraverser treeTraverser = new TreeTraverser() { + @Override + public Iterable children(FileDescriptorProto root) { + return root.getDependencyList() + .stream() + // Note: skip visiting dependencies if they have already been compiled + .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) + .map(protosByFileName::get) + .collect(Collectors.toSet()); + } + }; + for (FileDescriptorProto root : rootsByFileName.values()) { + for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + FileDescriptor[] dependencies = proto.getDependencyList() + .stream() + .map(compiledProtosByFileName::get) + .toArray(FileDescriptor[]::new); + + FileDescriptor compiledProto; + try { + compiledProto = FileDescriptor.buildFrom(proto, dependencies); + } catch (DescriptorValidationException e) { + log.warn("Exception encountered when building proto... skipping", e); + continue; + } + + compiledProtosByFileName.put(proto.getName(), compiledProto); + } + } + + return compiledProtosByFileName.values(); + } + + private void buildIndex( + Set availableServices, + Collection compiledProtos) { + JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); + for (FileDescriptor compiledProto : compiledProtos) { + typeRegistryBuilder.add(compiledProto.getMessageTypes()); + for (Descriptors.ServiceDescriptor service : compiledProto.getServices()) { + if (availableServices.contains(service.getFullName())) { + for (Descriptors.MethodDescriptor method : service.getMethods()) { + String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); + availableMethodsByFullMethodName.put(methodFullName, AvailableMethod.builder() + .methodDescriptor(method) + .typeRegistry(null) // TODO this?? + .build()); + } + } + } + } + } + +} From 4e0ee859c688c3404b07a5d04fc06858579b80a0 Mon Sep 17 00:00:00 2001 From: Jared Newman Date: Sun, 10 Jun 2018 16:45:31 -0700 Subject: [PATCH 11/11] hmm --- .../grpc/bridge/ReflectionChannel.java | 4 +- .../grpc/bridge/ReflectionChannelImpl.java | 11 +- .../grpc/bridge/ServiceIndexImpl.java | 147 +++++++++++++++++- .../grpc/bridge/ServiceIndexImpl2.java | 130 ---------------- .../bridge/ReflectionChannelImplTest.java | 6 +- .../grpc/bridge/ServiceIndexImplTest.java | 16 ++ 6 files changed, 174 insertions(+), 140 deletions(-) delete mode 100644 grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java create mode 100644 grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ServiceIndexImplTest.java diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java index 7d77a3b..5b9dddc 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannel.java @@ -5,7 +5,6 @@ package com.github.jared2501.grpc.bridge; import com.google.protobuf.DescriptorProtos; -import io.grpc.reflection.v1alpha.ServerReflectionGrpc; public interface ReflectionChannel { interface ReflectionObserver { @@ -36,6 +35,5 @@ interface ReflectionCall { void cancel(Throwable reason); } - ReflectionCall startCall( - String serviceName, ServerReflectionGrpc.ServerReflectionStub reflectionStub, ReflectionObserver observer); + ReflectionCall startCall(ReflectionObserver observer); } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java index 22b8a1a..b0049bb 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ReflectionChannelImpl.java @@ -24,9 +24,16 @@ public final class ReflectionChannelImpl implements ReflectionChannel { private static final Logger log = LoggerFactory.getLogger(ReflectionChannelImpl.class); + private final ServerReflectionGrpc.ServerReflectionStub reflectionStub; + private final String serviceName; + + public ReflectionChannelImpl(ServerReflectionGrpc.ServerReflectionStub reflectionStub, String serviceName) { + this.reflectionStub = reflectionStub; + this.serviceName = serviceName; + } + @Override - public ReflectionCall startCall( - String serviceName, ServerReflectionGrpc.ServerReflectionStub reflectionStub, ReflectionObserver observer) { + public ReflectionCall startCall(ReflectionObserver observer) { Context.CancellableContext context = Context.CancellableContext.current().withCancellation(); context.run(() -> { ReflectionResponseObserver streamObs = new ReflectionResponseObserver(serviceName, observer); diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java index ff77cd5..6986aba 100644 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java +++ b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl.java @@ -4,29 +4,172 @@ package com.github.jared2501.grpc.bridge; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.collect.TreeTraverser; import com.google.common.util.concurrent.AbstractService; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.Descriptors; +import com.google.protobuf.util.JsonFormat; +import java.time.Duration; +import java.util.Collection; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +// TODO: re-index when the channel changes state from closed -> open public final class ServiceIndexImpl extends AbstractService implements ServiceIndex { + private static final Logger log = LoggerFactory.getLogger(ServiceIndexImpl.class); + private final ReflectionChannel reflectionChannel; + private final ScheduledExecutorService executorService; + private final Duration failureDelay; + + private AtomicReference currentCall = + new AtomicReference<>(); + private AtomicReference> index = + new AtomicReference<>(ImmutableMap.of()); - public ServiceIndexImpl() { + public ServiceIndexImpl( + ReflectionChannel reflectionChannel, ScheduledExecutorService executorService, Duration failureDelay) { + this.reflectionChannel = reflectionChannel; + this.executorService = executorService; + this.failureDelay = failureDelay; } @Override protected void doStart() { + scheduleReflection(Duration.ZERO); notifyStarted(); } @Override protected void doStop() { + ReflectionChannel.ReflectionCall call = currentCall.get(); + if (call != null) { + call.cancel(new RuntimeException()); + } notifyStopped(); } @Override public Optional getMethod(String fullMethodName) { - return Optional.empty(); + return Optional.ofNullable(index.get().get(fullMethodName)); + } + + private void scheduleReflection(Duration scheduleDelay) { + executorService.schedule( + () -> { + if (!isRunning()) { + return; + } + ReflectionChannel.ReflectionCall lastCall = currentCall.getAndSet( + reflectionChannel.startCall(new IndexFromReflectionObserver())); + lastCall.cancel(new RuntimeException()); + }, + scheduleDelay.toMillis(), + TimeUnit.MILLISECONDS); + } + + private class IndexFromReflectionObserver implements ReflectionChannel.ReflectionObserver { + + private Set availableServices = Sets.newHashSet(); + private Map protoFiles = Maps.newHashMap(); + + @Override + public void onAvailableService(String serviceName) { + availableServices.add(serviceName); + } + + @Override + public void onProtoFile(String fileName, DescriptorProtos.FileDescriptorProto proto) { + protoFiles.put(fileName, proto); + } + + @Override + public void onComplete() { + index.set(buildIndex(compileProtos())); + } + + @Override + public void onError(Throwable error) { + log.warn("Error encountered receiving server reflection. Restart reflection. delay={}", + failureDelay, error); + scheduleReflection(failureDelay); + } + + + private Collection compileProtos() { + // Find all "roots", where a root is a proto file for which another proto file does not depend on it + Map rootsByFileName = Maps.newHashMap(protoFiles); + for (DescriptorProtos.FileDescriptorProto proto : protoFiles.values()) { + for (String dependencyFileName : proto.getDependencyList()) { + rootsByFileName.remove(dependencyFileName); + } + } + + Map compiledProtosByFileName = Maps.newHashMap(); + + // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto + // file if it has not already been compiled + TreeTraverser treeTraverser = new TreeTraverser() { + @Override + public Iterable children( + DescriptorProtos.FileDescriptorProto root) { + return root.getDependencyList() + .stream() + // Note: skip visiting dependencies if they have already been compiled + .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) + .map(protoFiles::get) + .collect(Collectors.toSet()); + } + }; + for (DescriptorProtos.FileDescriptorProto root : rootsByFileName.values()) { + for (DescriptorProtos.FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { + Descriptors.FileDescriptor[] dependencies = proto.getDependencyList() + .stream() + .map(compiledProtosByFileName::get) + .toArray(Descriptors.FileDescriptor[]::new); + + Descriptors.FileDescriptor compiledProto; + try { + compiledProto = Descriptors.FileDescriptor.buildFrom(proto, dependencies); + } catch (Descriptors.DescriptorValidationException e) { + log.warn("Exception encountered when building proto... skipping", e); + continue; + } + + compiledProtosByFileName.put(proto.getName(), compiledProto); + } + } + + return compiledProtosByFileName.values(); + } + + private Map buildIndex( + Collection compiledProtos) { + Map index = Maps.newHashMap(); + JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); + for (Descriptors.FileDescriptor compiledProto : compiledProtos) { + typeRegistryBuilder.add(compiledProto.getMessageTypes()); + for (Descriptors.ServiceDescriptor service : compiledProto.getServices()) { + if (availableServices.contains(service.getFullName())) { + for (Descriptors.MethodDescriptor method : service.getMethods()) { + String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); + index.put(methodFullName, method); + } + } + } + } + return index; + } } } diff --git a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java b/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java deleted file mode 100644 index 00f53de..0000000 --- a/grpc-json-bridge/src/main/java/com/github/jared2501/grpc/bridge/ServiceIndexImpl2.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. - */ - -package com.github.jared2501.grpc.bridge; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.collect.TreeTraverser; -import com.google.protobuf.DescriptorProtos.FileDescriptorProto; -import com.google.protobuf.Descriptors; -import com.google.protobuf.Descriptors.DescriptorValidationException; -import com.google.protobuf.Descriptors.FileDescriptor; -import com.google.protobuf.util.JsonFormat; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class ServiceIndexImpl2 implements ServiceIndex { - - private static final Logger log = LoggerFactory.getLogger(ServiceIndexImpl2.class); - - private final Set availableServices = Sets.newHashSet(); - private final Map protosByFileName = Maps.newHashMap(); - - private final Map availableMethodsByFullMethodName = Maps.newHashMap(); - private boolean isComplete = false; - - public void restart() { - availableServices.clear(); - protosByFileName.clear(); - } - - public void addAvailableService(String serviceName) { - availableServices.add(serviceName); - } - - public void addProto(String fileName, FileDescriptorProto proto) { - protosByFileName.put(fileName, proto); - } - - public void complete() { - isComplete = true; - buildIndex(availableServices, compileProtos(protosByFileName)); - } - - public boolean isAvailable() { - return isComplete; - } - - public boolean containsProto(String fileName) { - return protosByFileName.containsKey(fileName); - } - - @Override - public Optional getMethod(String fullMethodName) { - return Optional.ofNullable(availableMethodsByFullMethodName.get(fullMethodName)); - } - - private static Collection compileProtos(Map protosByFileName) { - // Find all "roots", where a root is a proto file for which another proto file does not depend on it - Map rootsByFileName = Maps.newHashMap(protosByFileName); - for (FileDescriptorProto proto : protosByFileName.values()) { - for (String dependencyFileName : proto.getDependencyList()) { - rootsByFileName.remove(dependencyFileName); - } - } - - Map compiledProtosByFileName = Maps.newHashMap(); - - // Perform a postorder traversal (i.e. visit children first) from every root, compiling and storing the proto - // file if it has not already been compiled - TreeTraverser treeTraverser = new TreeTraverser() { - @Override - public Iterable children(FileDescriptorProto root) { - return root.getDependencyList() - .stream() - // Note: skip visiting dependencies if they have already been compiled - .filter(dependencyFileName -> !compiledProtosByFileName.containsKey(dependencyFileName)) - .map(protosByFileName::get) - .collect(Collectors.toSet()); - } - }; - for (FileDescriptorProto root : rootsByFileName.values()) { - for (FileDescriptorProto proto : treeTraverser.postOrderTraversal(root)) { - FileDescriptor[] dependencies = proto.getDependencyList() - .stream() - .map(compiledProtosByFileName::get) - .toArray(FileDescriptor[]::new); - - FileDescriptor compiledProto; - try { - compiledProto = FileDescriptor.buildFrom(proto, dependencies); - } catch (DescriptorValidationException e) { - log.warn("Exception encountered when building proto... skipping", e); - continue; - } - - compiledProtosByFileName.put(proto.getName(), compiledProto); - } - } - - return compiledProtosByFileName.values(); - } - - private void buildIndex( - Set availableServices, - Collection compiledProtos) { - JsonFormat.TypeRegistry.Builder typeRegistryBuilder = JsonFormat.TypeRegistry.newBuilder(); - for (FileDescriptor compiledProto : compiledProtos) { - typeRegistryBuilder.add(compiledProto.getMessageTypes()); - for (Descriptors.ServiceDescriptor service : compiledProto.getServices()) { - if (availableServices.contains(service.getFullName())) { - for (Descriptors.MethodDescriptor method : service.getMethods()) { - String methodFullName = String.format("%s/%s", service.getFullName(), method.getName()); - availableMethodsByFullMethodName.put(methodFullName, AvailableMethod.builder() - .methodDescriptor(method) - .typeRegistry(null) // TODO this?? - .build()); - } - } - } - } - } - -} diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java index bfd63c6..14fab9e 100644 --- a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ReflectionChannelImplTest.java @@ -35,7 +35,7 @@ void setUp() throws IOException { .build() .start(); channel = InProcessChannelBuilder.forName("test").build(); - reflection = new ReflectionChannelImpl(); + reflection = new ReflectionChannelImpl(ServerReflectionGrpc.newStub(channel), "test"); } @AfterEach @@ -49,7 +49,7 @@ void tearDown() throws InterruptedException { @Test void successfulRoundTrip() { TestReflectionObserver observer = new TestReflectionObserver(); - reflection.startCall("test", ServerReflectionGrpc.newStub(channel), observer); + reflection.startCall(observer); observer.waitUntilComplete(); assertThat(observer.getError()).isNull(); @@ -68,7 +68,7 @@ void unavailableChannel() throws InterruptedException { channel.shutdown(); channel.awaitTermination(5, TimeUnit.HOURS); TestReflectionObserver observer = new TestReflectionObserver(); - reflection.startCall("test", ServerReflectionGrpc.newStub(channel), observer); + reflection.startCall(observer); observer.waitUntilComplete(); assertThat(observer.getError()) .isInstanceOf(StatusRuntimeException.class) diff --git a/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ServiceIndexImplTest.java b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ServiceIndexImplTest.java new file mode 100644 index 0000000..5ad14af --- /dev/null +++ b/grpc-json-bridge/src/test/java/com/github/jared2501/grpc/bridge/ServiceIndexImplTest.java @@ -0,0 +1,16 @@ +/* + * (c) Copyright 2018 Palantir Technologies Inc. All rights reserved. + */ + +package com.github.jared2501.grpc.bridge; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class ServiceIndexImplTest { + @Test + void getMethod() { + assertThat(true).isFalse(); + } +}