diff --git a/CHANGELOG.md b/CHANGELOG.md index 70fea34a9..9784db9f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Fixed +- Compile body to Array[Byte] only if request succeeded ## [v2.2.0] - 2020-04-04 ### Breaking changes diff --git a/docs-gh-pages/architecture.md b/docs-gh-pages/architecture.md deleted file mode 100644 index dc6a7c965..000000000 --- a/docs-gh-pages/architecture.md +++ /dev/null @@ -1,16 +0,0 @@ -## Architecture documentation -### Diagrams -#### ledger architecture -![ -Ledger Architecture -](img/architecture-diagrams/ledger-architecture.png) - -#### node-architecture -![ -Node architecture -](img/architecture-diagrams/node-architecture.png) - -#### protocol-data-flow-sketch -![ -Procotol Data Flow Sketch -](img/architecture-diagrams/protocol-data-flow-sketch.png) diff --git a/docs-gh-pages/design-choices.md b/docs-gh-pages/design-choices.md deleted file mode 100644 index 8253f4331..000000000 --- a/docs-gh-pages/design-choices.md +++ /dev/null @@ -1,35 +0,0 @@ -### Design choices - -This file summarizes current reasoning for our approach and design decisions, -reviews and references. - -Many decisions can be traced back to the fundamental goal to provide an -accessible, scalable protocol that focuses on solving the consensus task. -For a glimpse into the teams perspectives, you may check out the following video clips: - -* [youtube.com/constellation-labs/talk-at-Scale-By-The-Bay](https://youtu.be/iT5TjZGpajM) (Nov. 2018, 22 mins) -* [youtube.com/constellation-labs/talk-at-Tech-Crunch](https://youtu.be/fCscJL3_tdU) (Oct. 2018, 28 mins) -* [youtube.com/constellation-labs/testnet-overview](https://youtu.be/SsYZF4msXuQ) (Aug. 2018, 22 mins) - -#### Why Scala? -There are some general notes on Scala and also on other functional programming languages actively used for crypto projects in the -[/wiki/Comparisons-to-other-protocols](https://github.com/Constellation-Labs/constellation/wiki/Comparisons-to-other-protocols#fast_forward-projects-using-a-functional-language-approach). - -One motivating factor as a language of choice for the reference implementation of the protocol was of course the core teams experience with it, as well as the useful packages like akka actors and apache spark on the Java virtual machine (JVM). The constellation code base also makes extensive use of the type hierarchy features. In fact, the para-protocol approach to dApp integration builds on it. - -#### On the architecture - -For diagrams, see -[/docs/architecture.md](https://github.com/Constellation-Labs/constellation/blob/dev/docs/architecture.md). - -#### Feedback - -Please communicate suggestions by making a thread on the -[community portal Orion](https://orion.constellationlabs.io/accounts/login/?next=/) -or approaching the developers on the -[discord](https://discordapp.com/invite/KMSmXbV) -server: - - - - diff --git a/docs-gh-pages/img/architecture-diagrams/ledger-architecture.png b/docs-gh-pages/img/architecture-diagrams/ledger-architecture.png deleted file mode 100644 index 602cbe34d..000000000 Binary files a/docs-gh-pages/img/architecture-diagrams/ledger-architecture.png and /dev/null differ diff --git a/docs-gh-pages/img/architecture-diagrams/node-architecture.png b/docs-gh-pages/img/architecture-diagrams/node-architecture.png deleted file mode 100644 index 149af1b30..000000000 Binary files a/docs-gh-pages/img/architecture-diagrams/node-architecture.png and /dev/null differ diff --git a/docs-gh-pages/img/architecture-diagrams/protocol-data-flow-sketch.png b/docs-gh-pages/img/architecture-diagrams/protocol-data-flow-sketch.png deleted file mode 100644 index 810ea4054..000000000 Binary files a/docs-gh-pages/img/architecture-diagrams/protocol-data-flow-sketch.png and /dev/null differ diff --git a/docs-gh-pages/running-a-node.md b/docs-gh-pages/running-a-node.md index 5ea9a52c3..9b750c02e 100644 --- a/docs-gh-pages/running-a-node.md +++ b/docs-gh-pages/running-a-node.md @@ -1,12 +1,10 @@ # Running a node -Constellation nodes are currently released as JAR file. You can always find the latest one on -[constellation/releases](https://github.com/Constellation-Labs/constellation/releases) -page. -It can be run on any system supporting Java 8 or higher. ## Download release Releases are currently distributed as Java JAR files. -Download the [latest from github.](https://github.com/Constellation-Labs/constellation/releases) +Download the [latest from github](https://github.com/Constellation-Labs/constellation/releases) or from +[Dockerhub.](https://hub.docker.com/repository/docker/constellationprotocol/node) It can be run on any system +supporting Java 8 or higher. ## Node Requirements To run a constellation node, you'll need a machine with java 8 or higher installed. @@ -33,38 +31,27 @@ A constellation node has two API's: A control API meant for the node operator to By default, the control API uses port 9000, and the peer API uses port 9001. These can be overridden in the node configuration, but whatever they are set to, these ports must be open and exposed to the public internet. -## Connecting To An Existing Network -The most important configuration to connect to our testnet is providing a seed node to connect to. The easiest way to do this right now is to set it with an environment variable (example ip only, don't use): - -```export DAG_SEED_PEER=123.456.123.456:9001``` - -We are not currently running any publicly joinable network -- please get in touch on Telegram or Discord if you'd like to connect a node to our testnet network. - ## Where to Run ### Running a node at home -While running a from your home internet is possible, it's not a supported or recommended setup. It can be difficult to properly expose ports on your computer to the public internet, because home networking equipment like modems and routers often have firewalls blocking these ports, and often home networks do not have a stable ip address. For now, our official recommendation is to use a cloud provider. +While running a from your home is possible, it's not a supported or recommended setup. It can be difficult to properly +expose ports on your computer to the public internet, because home networking equipment like modems and routers often +have firewalls blocking these ports, and often home networks do not have a stable ip address. For now, our official +recommendation is to use a cloud provider. ### Cloud hosting -As a team we have the most expertise with Google Cloud, and it is our recommended platform for those without one already, but any cloud provider will work. -On -[Google Cloud Platform](https://cloud.google.com/) -(GCP), choose *compute engine* using a Ubuntu or Debian machine. -You can also do the installations up on the cloud machine. -Make sure the relevant ports are open / not firewalled. - -#### Cloud providers -Any cloud provider should work fine. While we do all our testing on GCP (Google Cloud Provider), we are not using any proprietary features. If you can launch a machine, install java 8 or higher, assign an external IP, and open the ports for the control API and peer API, you shouldn't have any problems. +Walkthrough for [GCP](https://docs.google.com/document/u/0/d/1KkJSdydz2DsAlQKaVB8MVqqQuPMr-ggTB2Y2niKOodY/mobilebasic) +Walkthrough for [AWS/DigitalOcean and Docker](https://docs.google.com/document/d/1qFYLqv2g_gHWWFs-x9YsPvrGzEocMPZS9hsJBegPrz0/edit) ## Manual Builds And Deployment -The [build instructions](https://github.com/Constellation-Labs/constellation/blob/dev/docs/build-instructions.md) have more pointers on how to run a node. +The [build instructions](https://github.com/Constellation-Labs/constellation/blob/dev/docs/build-instructions.md) +have more pointers on how to run a node. ### Additional Tools -Check out the -[recommended tools and frameworks](https://github.com/Constellation-Labs/constellation/blob/dev/docs/dependencies.md) -section in the docks. Here we highlight two: - -#### Docker -We intend to have official docker images soon -- stay tuned. +The DAG [Command Line Tool](https://github.com/zaemliss/Constellation) -#### Terraform (Optional!) -At Constellation Labs we use terraform to quickly launch and destroy clusters for testing. Our terraform configurations are checked in to the repository in the terraform directory, so they are available as a guide if you'd like to use it yourself. They automate some nice but optional things like running the node as a service. They are currently setup for GCP, and they have a few constellation-specific hardcoded variables (backend storage location, project name). That said -- if you're familiar with terraform, they should be straightforward to adapt for your uses. While we are not officially supporting this right now, we can provide some support on discord for this method. +## Connecting To An Existing Network +Be sure that you have the latest whitelist, which can be found by running +``wget https://github.com/Constellation-Labs/constellation/releases/latest/download/whitelisting`` +And you can update the whitelist with your info on [this spreadsheet](https://docs.google.com/spreadsheets/d/1MGBevI3MbhsN-oueC_q8ZPKRpWdPyaITcJpAhz60lPo/edit?pli=1#gid=0) +### Send join request +``curl -s -X POST http://YOURNODE/join -H "Content-type: application/json" -d "{ \"host\": \"DESTINATIONNODE\", \"port\": 9001 }"`` \ No newline at end of file diff --git a/src/main/scala/org/constellation/ConstellationNode.scala b/src/main/scala/org/constellation/ConstellationNode.scala index 53018c5ed..3976147c2 100644 --- a/src/main/scala/org/constellation/ConstellationNode.scala +++ b/src/main/scala/org/constellation/ConstellationNode.scala @@ -163,9 +163,9 @@ object ConstellationNode extends IOApp { getWhitelistedPeerId = { (ip: IP) => dao.nodeConfig.whitelisting.get(ip).pure[IO] } - peerEndpoints = PeerAuthMiddleware.requestVerifierMiddleware(getWhitelistedPeerId)(peerPublicEndpoints) <+> + peerEndpoints = PeerAuthMiddleware.requestVerifierMiddleware(getWhitelistedPeerId, false)(peerPublicEndpoints) <+> PeerAuthMiddleware - .requestVerifierMiddleware(getKnownPeerId)( + .requestVerifierMiddleware(getKnownPeerId, true)( PeerAuthMiddleware.whitelistingMiddleware(dao.nodeConfig.whitelisting, getKnownPeerId)( peerWhitelistedEndpoints ) diff --git a/src/main/scala/org/constellation/infrastructure/endpoints/middlewares/PeerAuthMiddleware.scala b/src/main/scala/org/constellation/infrastructure/endpoints/middlewares/PeerAuthMiddleware.scala index 99e81cf81..030017132 100644 --- a/src/main/scala/org/constellation/infrastructure/endpoints/middlewares/PeerAuthMiddleware.scala +++ b/src/main/scala/org/constellation/infrastructure/endpoints/middlewares/PeerAuthMiddleware.scala @@ -4,6 +4,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect.concurrent.Ref import cats.effect.{Concurrent, ContextShift, Resource, Sync} import cats.implicits._ +import fs2.{Chunk, Stream} import org.constellation.keytool.KeyUtils import org.constellation.primitives.IPManager.IP import org.constellation.schema.Id @@ -11,6 +12,7 @@ import org.http4s.Uri.{Authority, RegName, Scheme} import org.http4s._ import org.http4s.dsl.io._ import org.http4s.client.Client +import org.http4s.client.middleware.Logger import org.http4s.headers.Host import pl.abankowski.httpsigner.SignatureValid import pl.abankowski.httpsigner.http4s.{ @@ -23,7 +25,10 @@ import pl.abankowski.httpsigner.signature.generic.GenericVerifier object PeerAuthMiddleware { - def whitelistingMiddleware[F[_]: Sync](whitelisting: Map[IP, Id], knownPeer: IP => F[Option[Id]])( + def whitelistingMiddleware[F[_]: Sync]( + whitelisting: Map[IP, Id], + knownPeer: IP => F[Option[Id]] + )( http: HttpRoutes[F] ): HttpRoutes[F] = Kleisli { (req: Request[F]) => @@ -36,7 +41,12 @@ object PeerAuthMiddleware { .liftF(isWhitelisted) .ifM( http(req), { - OptionT.pure[F](Response(status = Unauthorized)) + OptionT.pure[F]( + Response(status = Unauthorized) + .withHeaders( + Header("Middleware-Result", s"Peer IP $ip is not whitelisted.") + ) + ) } ) } @@ -74,7 +84,11 @@ object PeerAuthMiddleware { ) }.flatMap(verifier.verify).flatMap { case SignatureValid => F.pure(response.withBodyStream(newBody)) - case _ => F.pure(Response[F](status = Unauthorized)) + case _ => + F.pure( + Response[F](status = Unauthorized) + .withHeaders(Header("Middleware-Result", "Invalid response signature")) + ) } } } @@ -88,7 +102,8 @@ object PeerAuthMiddleware { } def requestVerifierMiddleware[F[_]: Sync]( - knownPeer: IP => F[Option[Id]] + knownPeer: IP => F[Option[Id]], + usingKnownPeers: Boolean )(http: HttpRoutes[F])(implicit C: ContextShift[F]): HttpRoutes[F] = Kleisli { (req: Request[F]) => val ip = req.remoteAddr.getOrElse("unknown") @@ -103,11 +118,26 @@ object PeerAuthMiddleware { OptionT.liftF(verifier.verify(req)).flatMap { case SignatureValid => http(req) case sig => { - OptionT.pure[F](responseOnError) + OptionT.pure[F]( + responseOnError + .withHeaders( + Header("Middleware-Result", s"Invalid request signature, usingKnownPeers=$usingKnownPeers") + ) + ) } } } - .getOrElse(OptionT.pure[F](responseOnError)) + .getOrElse( + OptionT.pure[F]( + responseOnError + .withHeaders( + Header( + "Middleware-Result", + s"ID of $ip doesn't exist (Peer is not on PeerList probably). usingKnownPeers=$usingKnownPeers" + ) + ) + ) + ) } } } diff --git a/src/main/scala/org/constellation/infrastructure/p2p/client/SnapshotClientInterpreter.scala b/src/main/scala/org/constellation/infrastructure/p2p/client/SnapshotClientInterpreter.scala index c4c3dcbbb..0b5d926d4 100644 --- a/src/main/scala/org/constellation/infrastructure/p2p/client/SnapshotClientInterpreter.scala +++ b/src/main/scala/org/constellation/infrastructure/p2p/client/SnapshotClientInterpreter.scala @@ -1,23 +1,21 @@ package org.constellation.infrastructure.p2p.client import cats.effect.{Concurrent, ContextShift} -import io.circe.{Decoder, KeyDecoder} -import org.constellation.infrastructure.p2p.PeerResponse -import org.constellation.infrastructure.p2p.PeerResponse.PeerResponse -import org.http4s.client.Client import io.circe.generic.auto._ -import org.constellation.domain.observation.ObservationEvent +import io.circe.{Decoder, KeyDecoder} import org.constellation.domain.p2p.client.SnapshotClientAlgebra import org.constellation.domain.redownload.RedownloadService.{ LatestMajorityHeight, SnapshotProposalsAtHeight, SnapshotsAtHeight } +import org.constellation.infrastructure.p2p.PeerResponse +import org.constellation.infrastructure.p2p.PeerResponse.PeerResponse import org.constellation.schema.Id -import org.http4s.{EntityDecoder, MediaType} -import org.http4s.circe.CirceEntityDecoder._ -import org.http4s.circe.CirceEntityEncoder._ import org.http4s.Method._ +import org.http4s.Status.Successful +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.client.Client import scala.collection.SortedMap @@ -32,7 +30,10 @@ class SnapshotClientInterpreter[F[_]: Concurrent: ContextShift](client: Client[F def getStoredSnapshot(hash: String): PeerResponse[F, Array[Byte]] = PeerResponse[F, Vector[Byte]](s"snapshot/stored/$hash", client, GET) { (req, c) => - c.get(req.uri)(_.body.compile.toVector) + c.get(req.uri) { + case Successful(response) => response.body.compile.toVector + case response => Concurrent[F].raiseError(new Throwable(response.status.reason)) + } }.map(_.toArray) def getCreatedSnapshots(): PeerResponse[F, SnapshotProposalsAtHeight] = @@ -46,17 +47,22 @@ class SnapshotClientInterpreter[F[_]: Concurrent: ContextShift](client: Client[F def getSnapshotInfo(): PeerResponse[F, Array[Byte]] = // TODO: 45s timeout PeerResponse[F, Vector[Byte]](s"snapshot/info", client, GET) { (req, c) => - c.get(req.uri)(_.body.compile.toVector) + c.get(req.uri) { + case Successful(response) => response.body.compile.toVector + case response => Concurrent[F].raiseError(new Throwable(response.status.reason)) + } }.map(_.toArray) def getSnapshotInfo(hash: String): PeerResponse[F, Array[Byte]] = PeerResponse[F, Vector[Byte]](s"snapshot/info/$hash", client, GET) { (req, c) => - c.get(req.uri)(_.body.compile.toVector) + c.get(req.uri) { + case Successful(response) => response.body.compile.toVector + case response => Concurrent[F].raiseError(new Throwable(response.status.reason)) + } }.map(_.toArray) def getLatestMajorityHeight(): PeerResponse[F, LatestMajorityHeight] = PeerResponse[F, LatestMajorityHeight](s"latestMajorityHeight")(client) - } object SnapshotClientInterpreter {