diff --git a/cloud_install/aws/cloudformation-launch-stack.png b/cloud_install/aws/cloudformation-launch-stack.png new file mode 100644 index 000000000..7c0b53b6f Binary files /dev/null and b/cloud_install/aws/cloudformation-launch-stack.png differ diff --git a/cloud_install/aws/index.html b/cloud_install/aws/index.html new file mode 100644 index 000000000..d6e4f08c7 --- /dev/null +++ b/cloud_install/aws/index.html @@ -0,0 +1,13 @@ + + + +Mist + + + + + + + + + diff --git a/cloud_install/aws/install.sh b/cloud_install/aws/install.sh new file mode 100644 index 000000000..9a1c46101 --- /dev/null +++ b/cloud_install/aws/install.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +apt-get update +apt-get install -y nginx openjdk-8-jdk apache2-utils + +JAVA=$(which java) + +source ~/.aws_setup_data + +ssh-keygen -b 2048 -t rsa -f ~/.mist_key -q -N "" +chmod 600 ~/.mist_key + +echo "

Mist will start soon

" > /var/www/html/index.html +cat << EOF > /etc/nginx/sites-enabled/default +server { + listen 80 default_server; + listen [::]:80 default_server; + + root /var/www/html; + index index.html index.htm index.nginx-debian.html; + + server_name _; +} +EOF +service nginx restart + +cd /opt + +wget http://repo.hydrosphere.io/hydrosphere/static/preview/mist-1.0.0-RC18.tar.gz +tar xvfz mist-1.0.0-RC18.tar.gz +mv mist-1.0.0-RC18 mist + +wget https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz +tar xvfz spark-2.3.0-bin-hadoop2.7.tgz +mv spark-2.3.0-bin-hadoop2.7 spark + +# By some reasons, there is some problems with running Mist right after intance was started +# InfoProvider can't connect to master +sleep 30 + +INSTANCE_ID=$(ec2metadata --instance-id) +$JAVA -cp /opt/mist/utils/aws-init-setup.jar io.hydrosphere.mist.aws.Main $INSTANCE_ID $ACCESS_KEY_ID $ACCESS_KEY_SECRET $AWS_REGION /opt/mist/configs/default.conf ~/.mist_key.pub ~/.mist_key > /root/aws_setup.log + +SPARK_HOME=/opt/spark /opt/mist/bin/mist-master start + +htpasswd -b -c /etc/nginx/.htpasswd $MIST_LOGIN $MIST_PASSWORD +cat << EOF > /etc/nginx/sites-enabled/default +server { + listen 80 default_server; + listen [::]:80 default_server; + + root /var/www/html; + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location / { + proxy_pass http://127.0.0.1:2004/; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd; + } + + location /v2/api/ws { + proxy_pass http://127.0.0.1:2004/v2/api/ws; + proxy_http_version 1.1; + proxy_set_header Upgrade \$http_upgrade; + proxy_set_header Connection "upgrade"; + auth_basic "Restricted"; + auth_basic_user_file /etc/nginx/.htpasswd; + } +} +EOF +service nginx restart diff --git a/cloud_install/aws/template.json b/cloud_install/aws/template.json new file mode 100644 index 000000000..788b0bf29 --- /dev/null +++ b/cloud_install/aws/template.json @@ -0,0 +1,92 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Description" : "Hydrosphere/mist setup template", + "Metadata" : { + "Instances" : {"Description" : "Information about the instances"}, + "Databases" : {"Description" : "Information about the databases"} + }, + "Parameters" : { + "InstanceTypeParameter" : { + "Type" : "String", + "Default" : "m4.large", + "Description" : "Enter t2.micro, m1.small, m1.large ..." + }, + "KeyPairName": { + "Type": "AWS::EC2::KeyPair::KeyName", + "Description": "KeyPair Name" + }, + "AccessKeyID": { + "Type" : "String", + "Description" : "AWS access key ID" + }, + "AccessKeySecret": { + "Type" : "String", + "Description" : "AWS secret access key", + "NoEcho" : "true" + }, + "AZ": { + "Description": "Availability Zone of the Subnet", + "Type": "AWS::EC2::AvailabilityZone::Name" + }, + "MistLogin": { + "Type" : "String", + "Description" : "Basic auth login" + }, + "MistPassword": { + "Type" : "String", + "Description" : "Basic auth password", + "NoEcho" : "true" + } + }, + "Resources": { + "MistEc2Instance": { + "Type" : "AWS::EC2::Instance", + "Properties" : { + "AvailabilityZone" : { "Ref": "AZ" }, + "InstanceType": { "Ref": "InstanceTypeParameter" }, + "ImageId" : "ami-027583e616ca104df", + "KeyName": { "Ref": "KeyPairName" }, + "SecurityGroups": [ { "Ref": "InstanceSecurityGroup" } ], + "Tags" : [ {"Key" : "Name", "Value" : "mist-server"} ], + "UserData": { + "Fn::Base64": { "Fn::Join": [ "\n", [ + "#!/bin/bash -ex", + { "Fn::Join": ["", ["echo ACCESS_KEY_ID=", { "Ref": "AccessKeyID"}, " > ~/.aws_setup_data"]]}, + { "Fn::Join": ["", ["echo ACCESS_KEY_SECRET=", { "Ref": "AccessKeySecret"}, " >> ~/.aws_setup_data"]]}, + { "Fn::Join": ["", ["echo AWS_REGION=", { "Ref" : "AWS::Region" }, " >> ~/.aws_setup_data"]]}, + { "Fn::Join": ["", ["echo MIST_LOGIN=", { "Ref": "MistLogin"}, " >> ~/.aws_setup_data"]]}, + { "Fn::Join": ["", ["echo MIST_PASSWORD=", { "Ref": "MistPassword"}, " >> ~/.aws_setup_data"]]}, + "curl http://repo.hydrosphere.io/hydrosphere/static/preview/install.sh | bash" + ]]} + } + } + }, + "InstanceSecurityGroup" : { + "Type" : "AWS::EC2::SecurityGroup", + "Properties" : { + "GroupDescription" : "SSH + HTTP", + "SecurityGroupIngress" : [ + { + "IpProtocol" : "tcp", + "FromPort" : "22", + "ToPort" : "22", + "CidrIp" : "0.0.0.0/0" + }, + { + "IpProtocol" : "tcp", + "FromPort" : "80", + "ToPort" : "80", + "CidrIp" : "0.0.0.0/0" + } + ] + } + } + }, + "Outputs" : { + "PublicDns" : { + "Value" : { "Fn::GetAtt" : [ "MistEc2Instance", "PublicDnsName" ]}, + "Description" : "Public Dns name" + } + + } +} diff --git a/configs/logging/log4j.debug.properties b/configs/logging/log4j.debug.properties index d2379bd50..a24791123 100644 --- a/configs/logging/log4j.debug.properties +++ b/configs/logging/log4j.debug.properties @@ -9,3 +9,5 @@ log4j.appender.file.File=${mist.home}/logs/mist.log log4j.appender.file.append=true log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +log4j.logger.net.schmizz.sshj=WARN diff --git a/configs/logging/log4j.default.properties b/configs/logging/log4j.default.properties index 19d3737b0..02aa6b22f 100644 --- a/configs/logging/log4j.default.properties +++ b/configs/logging/log4j.default.properties @@ -4,4 +4,6 @@ log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file.File=${mist.home}/logs/mist.log log4j.appender.file.append=true log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +log4j.logger.net.schmizz.sshj=WARN diff --git a/examples/examples/src/main/scala/SparkSessionExample.scala b/examples/examples/src/main/scala/SparkSessionExample.scala index bd0cc67e1..368d4772c 100644 --- a/examples/examples/src/main/scala/SparkSessionExample.scala +++ b/examples/examples/src/main/scala/SparkSessionExample.scala @@ -11,6 +11,7 @@ object SparkSessionExample extends MistFn { arg[Int]("multiplier", 2) ).withMistExtras .onSparkSession((nums: Seq[Int], mult: Int, extras: MistExtras, spark: SparkSession) => { + spark.read.parquet("1", "2") spark.sparkContext.parallelize(nums).map(_ * mult).collect() }) diff --git a/mist.sbt b/mist.sbt index f958a2be9..7d2985fe9 100644 --- a/mist.sbt +++ b/mist.sbt @@ -50,13 +50,12 @@ lazy val mistLib = project.in(file("mist-lib")) test in Test := Def.sequential(test in Test, PyProject.pyTest in Test).value ) -lazy val core = project.in(file("mist/core")) +lazy val common = project.in(file("mist/common")) .dependsOn(mistLib) .settings(commonSettings: _*) .settings( - name := "mist-core", + name := "mist-common", scalacOptions ++= commonScalacOptions, - libraryDependencies ++= Library.spark(sparkVersion.value).map(_ % "runtime"), libraryDependencies ++= Seq( Library.Akka.actor, Library.slf4j, @@ -67,7 +66,7 @@ lazy val core = project.in(file("mist/core")) ) lazy val master = project.in(file("mist/master")) - .dependsOn(core % "compile->compile;test->test") + .dependsOn(common % "compile->compile;test->test") .settings(commonSettings: _*) .settings(commonAssemblySettings: _*) .enablePlugins(BuildInfoPlugin) @@ -76,17 +75,20 @@ lazy val master = project.in(file("mist/master")) scalacOptions ++= commonScalacOptions, libraryDependencies ++= Library.Akka.base, libraryDependencies ++= Seq( - Library.slf4jLog4j, Library.typesafeConfig, Library.scopt, + Library.slf4jLog4j, Library.log4j, Library.log4jExtras, + Library.typesafeConfig, Library.scopt, Library.slick, Library.h2, Library.flyway, Library.chill, Library.kafka, Library.pahoMqtt, Library.Akka.testKit % "test", Library.Akka.http, Library.Akka.httpSprayJson, Library.Akka.httpTestKit % "test", - Library.cats, + Library.cats, Library.catsEffect, Library.dockerJava, + Library.awsSdkEC2, Library.awsSdkEMR, Library.scalaSsh, + "io.hydrosphere" %% "shadedshapeless" % "2.3.0", Library.commonsCodec, Library.scalajHttp, Library.jsr305 % "provided", @@ -100,7 +102,7 @@ lazy val master = project.in(file("mist/master")) ) lazy val worker = project.in(file("mist/worker")) - .dependsOn(core % "compile->compile;test->test") + .dependsOn(common % "compile->compile;test->test") .settings(commonSettings: _*) .settings(commonAssemblySettings: _*) .settings( @@ -131,8 +133,45 @@ lazy val worker = project.in(file("mist/worker")) ) ) +lazy val agent = project.in(file("mist/agent")) + .dependsOn(common % "compile->compile;test->test") + .settings(commonSettings: _*) + .settings(commonAssemblySettings: _*) + .enablePlugins(BuildInfoPlugin) + .settings( + name := "mist-agent", + scalacOptions ++= commonScalacOptions, + libraryDependencies ++= Library.Akka.base, + libraryDependencies ++= Seq( + Library.slf4jLog4j, Library.log4j, Library.log4jExtras, + Library.Akka.testKit % "test", + + Library.awsSdkEMR, + Library.scalaTest % "test" + ) + ).settings( + buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sparkVersion), + buildInfoPackage := "io.hydrosphere.mist" +) + +lazy val awsInitSetup = project.in(file("mist/aws-init-setup")) + .dependsOn(common % "compile->compile;test->test") + .settings(commonSettings: _*) + .settings(commonAssemblySettings: _*) + .settings( + name := "mist-aws-init-setup", + scalacOptions ++= commonScalacOptions, + libraryDependencies ++= Seq( + Library.slf4jLog4j, Library.typesafeConfig, + Library.cats, Library.catsEffect, + Library.awsSdkEC2, Library.awsSdkIAM, + Library.jsr305 % "provided", + Library.scalaTest % "test" + ) + ) + lazy val root = project.in(file(".")) - .aggregate(mistLib, core, master, worker, examples) + .aggregate(mistLib, common, master, worker, examples) .dependsOn(master) .enablePlugins(DockerPlugin) .settings(commonSettings: _*) @@ -150,6 +189,9 @@ lazy val root = project.in(file(".")) CpFile("configs/logging").to("configs"), CpFile(assembly.in(master, assembly).value).as("mist-master.jar"), CpFile(assembly.in(worker, assembly).value).as("mist-worker.jar"), + CpFile(assembly.in(agent, assembly).value).as("mist-agent.jar"), + MkDir("utils"), + CpFile(assembly.in(awsInitSetup, assembly).value).as("aws-init-setup.jar").to("utils"), CpFile(Ui.ui.value).as("ui") ) }, diff --git a/mist/agent/src/main/resources/agent.conf b/mist/agent/src/main/resources/agent.conf new file mode 100644 index 000000000..a2c48ca2c --- /dev/null +++ b/mist/agent/src/main/resources/agent.conf @@ -0,0 +1,27 @@ +akka { + + loggers = ["akka.event.slf4j.Slf4jLogger"] + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + + loglevel = "INFO" + actor { + provider = "akka.remote.RemoteActorRefProvider" + warn-about-java-serializer-usage = false + } + + remote { + netty.tcp { + port = 0 + maximum-frame-size = 5242880b + } + transport-failure-detector { + heartbeat-interval = 30s + acceptable-heartbeat-pause = 5s + } + log-remote-lifecycle-events = off + } + + http.server.transparent-head-requests = false + http.server.idle-timeout = infinite + +} diff --git a/mist/agent/src/main/resources/log4j.properties b/mist/agent/src/main/resources/log4j.properties new file mode 100644 index 000000000..3d9d43a4e --- /dev/null +++ b/mist/agent/src/main/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala new file mode 100644 index 000000000..a247849d6 --- /dev/null +++ b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/ClusterAgent.scala @@ -0,0 +1,58 @@ +package io.hydrosphere.mist.agent + +import akka.actor.{ActorRef, ActorSystem} +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.utils.NetUtils +import io.hydrosphere.mist.utils.akka.{ActorRegHub, WhenTerminated} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +object ClusterAgent { + + def main(args: Array[String]): Unit = { + val masterAddr = args(0) + val id = args(1) + val accessKey = args(2) + val secretKey = args(3) + val region = args(4) + val awsId = args(5) + + val termination = EMRTermination.create(accessKey, secretKey, region) + + val hostname = NetUtils.findLocalInetAddress().getHostAddress + + val config = ConfigFactory.load("agent") + .withValue("akka.remote.netty.tcp.hostname", ConfigValueFactory.fromAnyRef(hostname)) + + implicit val system = ActorSystem("mist-agent", config) + + def resolveRemote(path: String): ActorRef = { + val ref = system.actorSelection(path).resolveOne(10 seconds) + try { + Await.result(ref, Duration.Inf) + } catch { + case e: Throwable => + println(s"Couldn't resolve remote path $path") + e.printStackTrace() + sys.exit(-1) + } + } + + def remotePath(addr: String, name: String): String = { + s"akka.tcp://mist@$addr/user/$name" + } + + val regHub = resolveRemote(remotePath(masterAddr, "regHub")) + val heathRef = resolveRemote(remotePath(masterAddr, CommonData.HealthActorName)) + regHub ! ActorRegHub.Register(id) + + WhenTerminated(heathRef, { + println("Remote system was terminated, shutdown cluster") + Await.result(termination.terminate(awsId), Duration.Inf) + system.terminate() + }) + + } +} diff --git a/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala new file mode 100644 index 000000000..189409010 --- /dev/null +++ b/mist/agent/src/main/scala/io/hydrosphere/mist/agent/EMRTermination.scala @@ -0,0 +1,34 @@ +package io.hydrosphere.mist.agent + +import software.amazon.awssdk.services.emr.EMRAsyncClient +import software.amazon.awssdk.services.emr.model.TerminateJobFlowsRequest + +import scala.concurrent.Future +import scala.concurrent.ExecutionContext.Implicits.global +import io.hydrosphere.mist.utils.CFConversion._ +import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region + +class EMRTermination(client: EMRAsyncClient) { + + def terminate(id: String): Future[Unit] = { + val req = TerminateJobFlowsRequest.builder().jobFlowIds(id).build() + client.terminateJobFlows(req).toFuture.map(_ => ()) + } + +} + +object EMRTermination { + + def create(accessKey: String, secretKey: String, region: String): EMRTermination = { + val credentials = AwsCredentials.create(accessKey, secretKey) + val provider = StaticCredentialsProvider.create(credentials) + val reg = Region.of(region) + val emrClient = EMRAsyncClient.builder() + .credentialsProvider(provider) + .region(reg) + .build() + + new EMRTermination(emrClient) + } +} diff --git a/mist/aws-init-setup/src/main/resources/log4j.properties b/mist/aws-init-setup/src/main/resources/log4j.properties new file mode 100644 index 000000000..49025057f --- /dev/null +++ b/mist/aws-init-setup/src/main/resources/log4j.properties @@ -0,0 +1,6 @@ +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + diff --git a/mist/aws-init-setup/src/main/resources/trustAutoScaling.json b/mist/aws-init-setup/src/main/resources/trustAutoScaling.json new file mode 100644 index 000000000..9ad2b0347 --- /dev/null +++ b/mist/aws-init-setup/src/main/resources/trustAutoScaling.json @@ -0,0 +1,16 @@ +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": [ + "elasticmapreduce.amazonaws.com", + "application-autoscaling.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} \ No newline at end of file diff --git a/mist/aws-init-setup/src/main/resources/trustEC2EMR.json b/mist/aws-init-setup/src/main/resources/trustEC2EMR.json new file mode 100644 index 000000000..37530dacb --- /dev/null +++ b/mist/aws-init-setup/src/main/resources/trustEC2EMR.json @@ -0,0 +1,15 @@ +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} \ No newline at end of file diff --git a/mist/aws-init-setup/src/main/resources/trustEMR.json b/mist/aws-init-setup/src/main/resources/trustEMR.json new file mode 100644 index 000000000..5a4d3716d --- /dev/null +++ b/mist/aws-init-setup/src/main/resources/trustEMR.json @@ -0,0 +1,15 @@ +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": [ + "elasticmapreduce.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} \ No newline at end of file diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala new file mode 100644 index 000000000..a8efefd3b --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/AwsSetup.scala @@ -0,0 +1,90 @@ +package io.hydrosphere.mist.aws + +import cats._ +import cats.effect.IO +import cats.implicits._ +import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.ec2.EC2AsyncClient +import software.amazon.awssdk.services.iam.IAMAsyncClient + +case class SetupData( + subnetId: String, + securityGroupId: String, + emrRole: String, + ec2EmrRole: String, + autoScalingRole: String, + sshKeyPairName: String +) + +trait AwsSetup[F[_]] { + + def setup(instanceId: String, sshKey: String): F[SetupData] + +} + +object AwsSetup { + + def default[F[_]](iam: IAMService[F], ec2: EC2Service[F])(implicit ME: MonadError[F, Throwable]): AwsSetup[F] = { + new AwsSetup[F] { + + val ec2EmrRole = AWSRole("mist-EMREC2", "default emr ec2 role", AWSRoleData.EC2EMR) + val emrRole = AWSRole("mist-EMR", "default emr role", AWSRoleData.EMR) + val autoScalingRole = AWSRole("mist-EMR-Autoscaling", "default autoscalaing role for emr", AWSRoleData.EMRAutoScaling) + val secGroupDecr = "Master-worker communications" + + def secGroupName(id: String): String = s"mist-internal-$id" + def keyName(id: String): String = s"mist-$id" + + override def setup(instanceId: String, sshKey: String): F[SetupData] = { + for { + maybeInst <- ec2.getInstanceData(instanceId) + data <- maybeInst match { + case Some(d) => ME.pure(d) + case None => ME.raiseError[InstanceData](new RuntimeException(s"Unknown instance: $instanceId")) + } + // analog of `aws emr create-default-roles` + // for ec2 it's required to create InstanceProfile linked with Role + // for emr it's enough to have only role + ec2EmrRole <- iam.getOrCreateRole(ec2EmrRole) + ec2EmrInstanceProfile <- iam.getOrCreateInstanceProfile(ec2EmrRole.name, ec2EmrRole.name) + emrRole <- iam.getOrCreateRole(emrRole) + autoScalingRole <- iam.getOrCreateRole(autoScalingRole) + + // additional security group for emr + // allows ingress traffic from mist-master + secGroupData = IngressData(0, 65535, IngressAddr.CidrIP(data.cidrIp), "TCP") + scGroupName = secGroupName(instanceId) + internalSecGroup <- ec2.getOrCreateSecGroup(scGroupName, secGroupDecr, data.vpcId, secGroupData) + + // add ingress rule to mist-master node security group + // allows ingress traffic from emr cluster + // as source uses security group created above + secGroupId = internalSecGroup.id + _ <- ec2.addIngressRule(data.secGroupIds.head, IngressData(0, 65535, IngressAddr.Group(internalSecGroup), "TCP")) + + keyName <- ec2.getOrCreateKeyPair(keyName(instanceId), sshKey) + } yield SetupData(data.subnetId, secGroupId, emrRole.name, ec2EmrRole.name, autoScalingRole.name, keyName) + } + } + } + + def create(accessKey: String, secretKey: String, regionName: String): AwsSetup[IO] = { + val credentials = AwsCredentials.create(accessKey, secretKey) + val provider = StaticCredentialsProvider.create(credentials) + + val ec2Client = EC2AsyncClient.builder() + .credentialsProvider(provider) + .region(Region.of(regionName)) + .build() + + val iamClient = IAMAsyncClient.builder() + .credentialsProvider(provider) + .region(Region.AWS_GLOBAL) + .build() + + val iam = IAMService.fromSdk(iamClient) + val ec2 = EC2Service.fromSdk(ec2Client) + default(iam, ec2) + } +} diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala new file mode 100644 index 000000000..4f8d1b36b --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/CompletableFutureOps.scala @@ -0,0 +1,35 @@ +package io.hydrosphere.mist.aws + +import java.util.concurrent.CompletableFuture +import java.util.function.{BiConsumer, BiFunction} + +import cats.effect.IO + +import scala.concurrent.{Future, Promise} + +final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal { + + def toFuture: Future[A] = { + val p = Promise[A] + cf.whenComplete(new BiConsumer[A, Throwable] { + override def accept(res: A, err: Throwable): Unit = { + (Option(res), Option(err)) match { + case (Some(r), None) => p.success(r) + case (_, Some(e)) => + e match { + case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause) + case _ => p.failure(e) + } + case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information")) + } + }}) + p.future + } + + def toIO: IO[A] = IO.fromFuture(IO(toFuture)) + +} + +object JFutureSyntax { + implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf) +} diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala new file mode 100644 index 000000000..19271a3a7 --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/EC2Service.scala @@ -0,0 +1,175 @@ +package io.hydrosphere.mist.aws + +import cats._ +import cats.implicits._ +import cats.effect.IO +import software.amazon.awssdk.services.ec2.EC2AsyncClient +import software.amazon.awssdk.services.ec2.model._ + +import scala.collection.JavaConverters._ +import JFutureSyntax._ + +sealed trait IngressAddr +object IngressAddr { + final case class CidrIP(value: String) extends IngressAddr + final case class Group(value: SecGroup) extends IngressAddr +} + +case class IngressData( + fromPort: Int, + toPort: Int, + address: IngressAddr, + protocol: String +) + +case class SecGroup(id: String, name: String) + +case class InstanceData( + ip: String, + vpcId: String, + subnetId: String, + secGroupIds: Seq[String] +) { + def cidrIp: String = ip + "/32" +} + +trait EC2Service[F[_]] { + + def getInstanceData(id: String): F[Option[InstanceData]] + + def getKeyPair(name: String): F[Option[String]] + def createKeyPair(name: String, key: String): F[String] + + def addIngressRule(groupId: String, data: IngressData): F[Unit] + + def getOrCreateKeyPair(name: String, key: String)(implicit M: Monad[F]): F[String] = { + for { + curr <- getKeyPair(name) + out <- curr match { + case Some(_) => M.pure(name) + case None => createKeyPair(name, key) + } + } yield out + } + + def getSecGroup(name: String): F[Option[SecGroup]] + def createSecGroup(name: String, descr: String, vpcId: String, data: IngressData): F[SecGroup] + + def getOrCreateSecGroup(name: String, descr: String, vpcId: String, data: IngressData)(implicit M: Monad[F]): F[SecGroup] = { + for { + curr <- getSecGroup(name) + out <- curr match { + case Some(v) => M.pure(v) + case None => createSecGroup(name, descr, vpcId, data) + } + } yield out + } + +} + +object EC2Service { + + def fromSdk(ec2Client: EC2AsyncClient): EC2Service[IO] = { + new EC2Service[IO] { + + override def addIngressRule(groupId: String, data: IngressData): IO[Unit] = { + import data._ + val ipPremission = IpPermission.builder() + .fromPort(fromPort) + .toPort(toPort) + .ipProtocol(protocol) + + val withAddr = address match { + case IngressAddr.CidrIP(v) => + val range = IpRange.builder().cidrIp(v).build() + ipPremission.ipv4Ranges(range) + case IngressAddr.Group(v) => + val group = UserIdGroupPair.builder() + .groupId(v.id) + .description(s"${v.id}:${v.name}") + .build() + ipPremission.userIdGroupPairs(group) + } + + val req = AuthorizeSecurityGroupIngressRequest.builder() + .groupId(groupId) + .ipPermissions(withAddr.build()) + .build() + + ec2Client.authorizeSecurityGroupIngress(req).toIO.map(_ => ()) + .handleErrorWith(e => e match { + case m: software.amazon.awssdk.services.ec2.model.EC2Exception if m.getMessage.contains("already exists") => IO.pure(()) + case _ => IO.raiseError(e) + }) + } + + override def getSecGroup(name: String): IO[Option[SecGroup]] = { + val req = DescribeSecurityGroupsRequest.builder() + .filters(Filter.builder() + .name("group-name") + .values(name) + .build()) + .build() + + ec2Client.describeSecurityGroups(req).toIO + .map(_.securityGroups().asScala.headOption.map(s => SecGroup(s.groupId(), s.groupName()))) + .handleErrorWith(e => e match { + case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None) + case _ => IO.raiseError(e) + }) + } + + override def createSecGroup( + name: String, + descr: String, + vpcId: String, + data: IngressData + ): IO[SecGroup] = { + val createReq = CreateSecurityGroupRequest.builder() + .groupName(name) + .description(descr) + .vpcId(vpcId) + .build() + + for { + groupId <- ec2Client.createSecurityGroup(createReq).toIO.map(_.groupId()) + _ <- addIngressRule(groupId, data) + } yield SecGroup(groupId, name) + } + + override def getInstanceData(id: String): IO[Option[InstanceData]] = { + def extractData(resp: DescribeInstancesResponse): Option[InstanceData] = { + for { + reservation <- resp.reservations().asScala.headOption + instance <- reservation.instances().asScala.headOption + } yield { + val secGroupsIds = instance.securityGroups().asScala.map(sg => sg.groupId()) + InstanceData(instance.privateIpAddress(), instance.vpcId(), instance.subnetId(), secGroupsIds) + } + } + + val req = DescribeInstancesRequest.builder().instanceIds(id).build() + for { + resp <- ec2Client.describeInstances(req).toIO + data = extractData(resp) + } yield data + } + + override def getKeyPair(name: String): IO[Option[String]] = { + val req = DescribeKeyPairsRequest.builder().keyNames(name).build() + ec2Client.describeKeyPairs(req).toIO + .map(resp => resp.keyPairs().asScala.headOption.map(i => i.keyName())) + .handleErrorWith(e => e match { + case _: software.amazon.awssdk.services.ec2.model.EC2Exception => IO.pure(None) + case _ => IO.raiseError(e) + }) + } + + override def createKeyPair(name: String, key: String): IO[String] = { + val req = ImportKeyPairRequest.builder().keyName(name).publicKeyMaterial(key).build() + ec2Client.importKeyPair(req).toIO.map(resp => resp.keyName()) + } + + } + } +} diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala new file mode 100644 index 000000000..2b0fe134f --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/GenConfig.scala @@ -0,0 +1,65 @@ +package io.hydrosphere.mist.aws + +import java.nio.file.{Files, Path} + +import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions} + +case class LaunchData( + sshKeyPair: String, + sshKeyPath: String, + accessKey: String, + secretKey: String, + subnetId: String, + region: String, + additionalGroup: String, + emrRole: String, + emrEc2Role: String, + autoScalingRole: String +) + +object ConfigPatcher { + + def patch(mistConfig: Config, data: LaunchData): Config = { + import com.typesafe.config.ConfigValueFactory._ + import scala.collection.JavaConverters._ + + import data._ + + val configKeys = Map( + "name" -> "default_emr", + "type" -> "aws_emr", + "sshKeyPair" -> sshKeyPair, + "sshKeyPath" -> sshKeyPath, + "sshUser" -> "hadoop", + "accessKey"-> accessKey, + "secretKey" -> secretKey, + "subnetId" -> subnetId, + "region" -> region, + "additionalGroup" -> additionalGroup, + "emrRole" -> emrRole, + "emrEc2Role" -> emrEc2Role, + "autoScalingRole" -> autoScalingRole + ).map({case (k, v) => k -> fromAnyRef(v)}) + + val provisioner = fromMap(configKeys.asJava) + val entries = fromIterable(Seq(provisioner).asJava) + + mistConfig.withValue("mist.launchers-settings", entries) + } + + def patchFile(filePath: Path, data: LaunchData): Unit = { + val orig = ConfigFactory.parseFile(filePath.toFile) + val patched = patch(orig, data) + + val renderOpts = ConfigRenderOptions.defaults() + .setComments(false) + .setOriginComments(false) + .setJson(false) + .setFormatted(true) + + val rawConfig = patched.root().render(renderOpts) + Files.write(filePath, rawConfig.getBytes) + } + +} + diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala new file mode 100644 index 000000000..7b43653b9 --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/IAMService.scala @@ -0,0 +1,138 @@ +package io.hydrosphere.mist.aws + +import java.util.concurrent.CompletableFuture +import java.util.function.BiConsumer + +import cats._ +import cats.implicits._ +import cats.Monad +import cats.effect._ +import software.amazon.awssdk.services.iam.IAMAsyncClient +import software.amazon.awssdk.services.iam.model._ +import JFutureSyntax._ + +import scala.io.Source + +case class AWSRoleData( + trustPolicyJson: String, + permissionsArn: String +) + +object AWSRoleData { + + private def readResourceJson(name: String): String = { + val stream = getClass.getResourceAsStream(name) + Source.fromInputStream(stream).mkString.replace("\n", "") + } + + val EMR = AWSRoleData( + readResourceJson("/trustEMR.json"), + "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole" + ) + + val EC2EMR = AWSRoleData( + readResourceJson("/trustEC2EMR.json"), + permissionsArn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role" + ) + + val EMRAutoScaling = AWSRoleData( + readResourceJson("/trustAutoScaling.json"), + permissionsArn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" + ) +} + +case class AWSRole( + name: String, + description: String, + data: AWSRoleData +) + +trait IAMService[F[_]] { + + def createRole(role: AWSRole): F[AWSRole] + def getRole(name: String): F[Option[String]] + + def getOrCreateRole(role: AWSRole)(implicit m: Monad[F]): F[AWSRole] = { + for { + out <- getRole(role.name) + role <- out match { + case Some(_) => m.pure(role) + case None => createRole(role) + } + } yield role + } + + def createInstanceProfile(name: String, role: String): F[String] + def getInstanceProfile(name: String): F[Option[String]] + def getOrCreateInstanceProfile(name: String, role: String)(implicit M: Monad[F]): F[String] = { + for { + out <- getInstanceProfile(name) + role <- out match { + case Some(_) => M.pure(role) + case None => createInstanceProfile(name, role) + } + } yield role + } + +} + +object IAMService { + + def fromSdk(iamClient: IAMAsyncClient): IAMService[IO] = + new IAMService[IO] { + + override def getRole(name: String): IO[Option[String]] = { + val req = GetRoleRequest.builder().roleName(name).build() + + iamClient.getRole(req).toIO + .map(r => Option(r.role().roleName())) + .handleErrorWith({ + case _: software.amazon.awssdk.services.iam.model.NoSuchEntityException => IO.pure(None) + case e => IO.raiseError(e) + }) + } + + override def createRole(role: AWSRole): IO[AWSRole] = { + val createReq = CreateRoleRequest.builder() + .roleName(role.name) + .description(role.description) + .assumeRolePolicyDocument(role.data.trustPolicyJson) + .build() + + val attachReq = AttachRolePolicyRequest.builder() + .policyArn(role.data.permissionsArn) + .roleName(role.name) + .build() + + for { + _ <- iamClient.createRole(createReq).toIO + _ <- iamClient.attachRolePolicy(attachReq).toIO + } yield role + } + + override def createInstanceProfile(name: String, role: String): IO[String] = { + val createReq = CreateInstanceProfileRequest.builder() + .instanceProfileName(name) + .build() + + val addReq = AddRoleToInstanceProfileRequest.builder() + .instanceProfileName(name) + .roleName(role) + .build() + for { + _ <- iamClient.createInstanceProfile(createReq).toIO + _ <- iamClient.addRoleToInstanceProfile(addReq).toIO + } yield name + } + + override def getInstanceProfile(name: String): IO[Option[String]] = { + val req = GetInstanceProfileRequest.builder().instanceProfileName(name).build() + iamClient.getInstanceProfile(req).toIO + .map(resp => Option(resp.instanceProfile().instanceProfileName())) + .handleErrorWith({ + case _: NoSuchEntityException => IO.pure(None) + case e => IO.raiseError(e) + }) + } + } +} diff --git a/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala new file mode 100644 index 000000000..203ca4c8e --- /dev/null +++ b/mist/aws-init-setup/src/main/scala/io/hydrosphere/mist/aws/Main.scala @@ -0,0 +1,39 @@ +package io.hydrosphere.mist.aws + +import java.nio.file.Paths + +import scala.io.Source + +object Main { + + def main(args: Array[String]): Unit = { + val instanceId = args(0) + val accessKey = args(1) + val accessSecret = args(2) + val region = args(3) + val configPath = args(4) + + val sshKeyPathPub = args(5) + val sshKeyPath = args(6) + val sshKeyPub = Source.fromFile(Paths.get(sshKeyPathPub).toFile).mkString + + val setup = AwsSetup.create(accessKey, accessSecret, region) + val out = setup.setup(instanceId, sshKeyPub).unsafeRunSync() + + val launchData = LaunchData( + sshKeyPair = out.sshKeyPairName, + sshKeyPath = sshKeyPath, + accessKey = accessKey, + secretKey = accessSecret, + subnetId = out.subnetId, + region = region, + additionalGroup = out.securityGroupId, + emrRole = out.emrRole, + emrEc2Role = out.ec2EmrRole, + autoScalingRole = out.autoScalingRole + ) + + ConfigPatcher.patchFile(Paths.get(configPath), launchData) + } + +} diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala similarity index 98% rename from mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala index 8c244b807..64cb7dce5 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/core/CommonData.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/CommonData.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core +package io.hydrosphere.mist.common import akka.actor.ActorRef import mist.api.data.{JsData, JsMap} @@ -27,7 +27,6 @@ object CommonData { */ case class WorkerInitInfo( sparkConf: Map[String, String], - maxJobs: Int, downtime: Duration, streamingDuration: Duration, logService: String, diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala similarity index 94% rename from mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala index 8374a83cf..2f8634140 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/core/FunctionInfoData.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/FunctionInfoData.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core +package io.hydrosphere.mist.common import mist.api.UserInputArgument diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala similarity index 93% rename from mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala index fa2a91b94..4632b748d 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/core/PythonEntrySettings.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/PythonEntrySettings.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core +package io.hydrosphere.mist.common case class PythonEntrySettings( driver: String, diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala new file mode 100644 index 000000000..19bde387a --- /dev/null +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/AgentProtocol.scala @@ -0,0 +1,10 @@ +package io.hydrosphere.mist.common.logging + +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo + +object AgentProtocol { + + case class Register(id: String) + case class StartWorker(info: WorkerInitInfo) + +} diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala similarity index 90% rename from mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala index 863a4de8d..bc09079ac 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/Level.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/Level.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core.logging +package io.hydrosphere.mist.common.logging case class Level(value: Int, name: String) diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala similarity index 97% rename from mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala index b226275ec..d41509b98 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/core/logging/LogEvent.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/common/logging/LogEvent.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core.logging +package io.hydrosphere.mist.common.logging import java.io.{PrintWriter, StringWriter} import java.time.format.DateTimeFormatter diff --git a/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala new file mode 100644 index 000000000..b53136134 --- /dev/null +++ b/mist/common/src/main/scala/io/hydrosphere/mist/utils/CFConverion.scala @@ -0,0 +1,32 @@ +package io.hydrosphere.mist.utils + +import java.util.concurrent.CompletableFuture +import java.util.function.{BiConsumer, BiFunction} + +import scala.concurrent._ + +final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal { + + def toFuture: Future[A] = { + val p = Promise[A] + cf.whenComplete(new BiConsumer[A, Throwable] { + override def accept(res: A, err: Throwable): Unit = { + (Option(res), Option(err)) match { + case (Some(r), None) => p.success(r) + case (_, Some(e)) => + e match { + case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause) + case _ => p.failure(e) + } + case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information")) + } + }}) + p.future + } + + +} + +object CFConversion { + implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf) +} diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/Collections.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/Collections.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/Collections.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/Collections.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala similarity index 90% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala index 5ae394646..7e86727a6 100644 --- a/mist/core/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala +++ b/mist/common/src/main/scala/io/hydrosphere/mist/utils/ConfigUtils.scala @@ -18,9 +18,11 @@ object ConfigUtils { def getOptString(path: String): Option[String] = getOpt(path, _.getString(path)) def getOptInt(path: String): Option[Int] = getOpt(path, _.getInt(path)) + def getOptConfig(path: String): Option[Config] = getOpt(path, _.getConfig(path)) def getOpt[A](path: String, f: Config => A): Option[A] = if (c.hasPath(path)) Option(f(c)) else None + } } diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/EitherOps.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/FutureOps.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/Logger.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/Logger.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/Logger.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/Logger.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/NetUtils.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/TryLoad.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorF.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/ActorRegHub.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/RestartSupervisor.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/akka/WhenTerminated.scala diff --git a/mist/core/src/main/scala/io/hydrosphere/mist/utils/fs.scala b/mist/common/src/main/scala/io/hydrosphere/mist/utils/fs.scala similarity index 100% rename from mist/core/src/main/scala/io/hydrosphere/mist/utils/fs.scala rename to mist/common/src/main/scala/io/hydrosphere/mist/utils/fs.scala diff --git a/mist/core/src/test/resources/log4j.properties b/mist/common/src/test/resources/log4j.properties similarity index 100% rename from mist/core/src/test/resources/log4j.properties rename to mist/common/src/test/resources/log4j.properties diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala similarity index 98% rename from mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala rename to mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala index 8796b7309..264613db5 100644 --- a/mist/core/src/test/scala/io/hydrosphere/mist/core/MockitoSugar.scala +++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/MockitoSugar.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core +package io.hydrosphere.mist.common import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.{Answer, OngoingStubbing} diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala similarity index 93% rename from mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala rename to mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala index f3f5bdacc..9115646bf 100644 --- a/mist/core/src/test/scala/io/hydrosphere/mist/core/PythonEntrySettingsSpec.scala +++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/PythonEntrySettingsSpec.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core +package io.hydrosphere.mist.common import org.scalatest.{FunSpec, Matchers} diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala similarity index 96% rename from mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala rename to mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala index c352937fc..a80df1b5a 100644 --- a/mist/core/src/test/scala/io/hydrosphere/mist/core/logging/MistLoggingSpec.scala +++ b/mist/common/src/test/scala/io/hydrosphere/mist/common/logging/MistLoggingSpec.scala @@ -1,4 +1,4 @@ -package io.hydrosphere.mist.core.logging +package io.hydrosphere.mist.common.logging import java.time.format.DateTimeFormatter import java.time.{LocalDateTime, ZoneOffset} diff --git a/mist/core/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala b/mist/common/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala similarity index 100% rename from mist/core/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala rename to mist/common/src/test/scala/io/hydrosphere/mist/utils/akka/ActorRegSpec.scala diff --git a/mist/master/src/main/resources/master.conf b/mist/master/src/main/resources/master.conf index 1d4dfcab9..912466ec1 100644 --- a/mist/master/src/main/resources/master.conf +++ b/mist/master/src/main/resources/master.conf @@ -126,6 +126,8 @@ mist { } } + launchers-settings = [] + } writers-blocking-dispatcher { diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala index dbca2b906..29acb528c 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/JobDetails.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master -import io.hydrosphere.mist.core.CommonData.JobParams +import io.hydrosphere.mist.common.CommonData.JobParams import io.hydrosphere.mist.master.JobDetails.Status import io.hydrosphere.mist.master.Messages.StatusMessages._ import mist.api.data.JsData diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala index 473a917fc..fdae4bb07 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MainService.scala @@ -5,7 +5,7 @@ import java.util.UUID import akka.util.Timeout import cats.data._ import cats.implicits._ -import io.hydrosphere.mist.core.CommonData.Action +import io.hydrosphere.mist.common.CommonData.Action import io.hydrosphere.mist.master.JobDetails.Source.Async import io.hydrosphere.mist.master.data.ContextsStorage import io.hydrosphere.mist.master.execution.{ExecutionInfo, ExecutionService} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala index 715b441f2..357404192 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/MasterServer.scala @@ -10,12 +10,13 @@ import akka.stream.ActorAttributes.supervisionStrategy import akka.stream.ActorMaterializer import akka.stream.Supervision.resumingDecider import akka.stream.scaladsl.{Keep, Sink} -import io.hydrosphere.mist.core.CommonData +import io.hydrosphere.mist.common.CommonData import io.hydrosphere.mist.master.Messages.StatusMessages.SystemEvent import io.hydrosphere.mist.master.artifact.ArtifactRepository import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage} +import io.hydrosphere.mist.master.execution.workers.WorkerRunner import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter -import io.hydrosphere.mist.master.execution.{ExecutionService, SpawnSettings} +import io.hydrosphere.mist.master.execution.{ClusterRunner, ClustersService, ExecutionService, SpawnSettings} import io.hydrosphere.mist.master.interfaces.async._ import io.hydrosphere.mist.master.interfaces.http._ import io.hydrosphere.mist.master.jobs.{FunctionInfoProviderRunner, FunctionsService} @@ -23,7 +24,7 @@ import io.hydrosphere.mist.master.logging.{LogService, LogStreams} import io.hydrosphere.mist.master.security.KInitLauncher import io.hydrosphere.mist.master.store.H2JobsRepository import io.hydrosphere.mist.utils.Logger -import io.hydrosphere.mist.utils.akka.RestartSupervisor +import io.hydrosphere.mist.utils.akka.{ActorRegHub, RestartSupervisor} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.{Future, Promise} @@ -99,11 +100,11 @@ object MasterServer extends Logger { LogStreams.runService(host, port, logsPaths, streamer) } - def runExecutionService(logService: LogService): ExecutionService = { + def runClustersService(): ClustersService = { val logsDir = Paths.get(config.logs.dumpDirectory) - val workerRunner = WorkerStarter.create(config.workers, logsDir) + + val regHub = ActorRegHub("regHub", system) val spawnSettings = SpawnSettings( - runnerCmd = workerRunner, timeout = config.workers.runnerInitTimeout, readyTimeout = config.workers.readyTimeout, akkaAddress = s"${config.cluster.publicHost}:${config.cluster.port}", @@ -111,7 +112,16 @@ object MasterServer extends Logger { httpAddress = s"${config.http.publicHost }:${config.http.port}", maxArtifactSize = config.workers.maxArtifactSize ) - ExecutionService(spawnSettings, system, streamer, store, logService) + val defaultRunner = { + val starter = WorkerStarter.create(config.workers, logsDir) + ClusterRunner.legacy(spawnSettings, regHub, starter, system) + } + + ClustersService.create(config.mistHome, spawnSettings, regHub, config.launchersSettings, defaultRunner, system) + } + + def runExecutionService(clustersService: ClustersService, logService: LogService): ExecutionService = { + ExecutionService(clustersService, system, streamer, store, logService) } val artifactRepository = ArtifactRepository.create( @@ -146,7 +156,8 @@ object MasterServer extends Logger { contextsStorage, artifactRepository )(system.dispatcher) - executionService = runExecutionService(logService) + clustersService = runClustersService() + executionService = runExecutionService(clustersService, logService) masterService <- start("Main service", MainService.start( executionService, contextsStorage, diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala index 773592183..3e0441365 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/Messages.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.JobDetails.Source import mist.api.data.JsData diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala index 9261e0aad..f65fb6232 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/configs.scala @@ -256,6 +256,58 @@ object SecurityConfig { } +sealed trait LauncherSettings + +case class AWSEMRLaunchSettings( + sshKeyPair: String, + sshKeyPath: String, + sshUser: String, + accessKey: String, + secretKey: String, + subnetId: String, + region: String, + additionalGroup: String, + emrRole: String, + emrEc2Role: String, + autoScalingRole: String +) extends LauncherSettings + +object AWSEMRLaunchSettings { + + def apply(c: Config): AWSEMRLaunchSettings = { + AWSEMRLaunchSettings( + sshKeyPair = c.getString("sshKeyPair"), + sshKeyPath = c.getString("sshKeyPath"), + sshUser = c.getString("sshUser"), + accessKey = c.getString("accessKey"), + secretKey = c.getString("secretKey"), + subnetId = c.getString("subnetId"), + region = c.getString("region"), + additionalGroup = c.getString("additionalGroup"), + emrRole = c.getString("emrRole"), + emrEc2Role = c.getString("emrEc2Role"), + autoScalingRole = c.getString("autoScalingRole") + ) + } +} + +object LauncherSettings { + + def apply(c: Config): LauncherSettings = { + c.getString("type") match { + case "aws_emr" => AWSEMRLaunchSettings(c) + case x => throw new IllegalArgumentException(s"Unknown launcher settings type: $x") + } + } + + def extractAll(all: Seq[Config]): Map[String, LauncherSettings] = { + all.map(c => { + val name = c.getString("name") + name -> LauncherSettings(c) + }).toMap + } +} + case class MasterConfig( cluster: HostPortConfig, http: HttpConfig, @@ -272,6 +324,8 @@ case class MasterConfig( srcConfigPath: String, jobsSavePath: String, artifactRepositoryPath: String, + launchersSettings: Map[String, LauncherSettings], + mistHome: String, raw: Config ) @@ -314,6 +368,8 @@ object MasterConfig extends Logger { security = SecurityConfig.ifEnabled(mist.getConfig("security")), jobInfoProviderConfig = FunctionInfoProviderConfig(mist.getConfig("job-extractor")), srcConfigPath = filePath, + launchersSettings = LauncherSettings.extractAll(mist.getConfigList("launchers-settings")), + mistHome = mist.getString("work-directory"), raw = config ) } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala index 419b2715b..0f1714436 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/ConfigRepr.scala @@ -1,21 +1,26 @@ package io.hydrosphere.mist.master.data -import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory, ConfigValueType} -import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, NamedConfig, RunMode} +import com.typesafe.config._ +import io.hydrosphere.mist.master.interfaces.{EMRInstanceFormat, JsonCodecs} +import io.hydrosphere.mist.master.models._ import io.hydrosphere.mist.utils.ConfigUtils._ +import spray.json.{JsonFormat, JsonParser, ParserInput} import scala.concurrent.duration._ -trait ConfigRepr[A <: NamedConfig] { - +trait ConfigRepr[A] { def toConfig(a: A): Config - def fromConfig(config: Config): A +} + +trait NamedConfigRepr[A] extends ConfigRepr[A] { def fromConfig(name: String, config: Config): A = fromConfig(config.withValue("name", ConfigValueFactory.fromAnyRef(name))) + } +//TODO it's the same as jsonCodecs! object ConfigRepr { import scala.collection.JavaConverters._ @@ -24,12 +29,7 @@ object ConfigRepr { def toConfig: Config = repr.toConfig(a) } - implicit class FromCongixSyntax(config: Config){ - def to[A <: NamedConfig](implicit repr: ConfigRepr[A]): A = repr.fromConfig(config) - def to[A <: NamedConfig](name: String)(implicit repr: ConfigRepr[A]): A = repr.fromConfig(name, config) - } - - implicit val EndpointsRepr = new ConfigRepr[FunctionConfig] { + val EndpointsRepr = new NamedConfigRepr[FunctionConfig] { override def toConfig(a: FunctionConfig): Config = { import ConfigValueFactory._ @@ -51,7 +51,55 @@ object ConfigRepr { } } - implicit val ContextConfigRepr: ConfigRepr[ContextConfig] = new ConfigRepr[ContextConfig] { + def fromJsonFormat[A](jsonFormat: JsonFormat[A]): ConfigRepr[A] = new ConfigRepr[A] { + override def toConfig(a: A): Config = { + val s = jsonFormat.write(a).prettyPrint + ConfigFactory.parseString(s) + } + override def fromConfig(config: Config): A = { + val jsonString = config.root().render(ConfigRenderOptions.defaults().setJson(true).setComments(false).setOriginComments(false)) + val json = JsonParser(ParserInput(jsonString)) + jsonFormat.read(json) + } + } + + val EMRInstanceRepr: ConfigRepr[EMRInstance.Instance] = fromJsonFormat(EMRInstanceFormat.instanceF) + + val LaunchDataConfigRepr: ConfigRepr[LaunchData] = new ConfigRepr[LaunchData] { + + override def toConfig(a: LaunchData): Config = { + import ConfigValueFactory._ + + val (t, body) = a match { + case ServerDefault => "server-default" -> Map.empty[String, ConfigValue] + case awsEmr: AWSEMRLaunchData => + val instances = awsEmr.instances.map(i => EMRInstanceRepr.toConfig(i).root()) + "aws-emr" -> Map( + "launcher-settings-name" -> fromAnyRef(awsEmr.launcherSettingsName), + "release-label" -> fromAnyRef(awsEmr.releaseLabel), + "instances" -> fromIterable(instances.asJava) + ) + } + val full = body + ("type" -> t) + fromMap(full.asJava).toConfig + } + + override def fromConfig(config: Config): LaunchData = { + config.getString("type") match { + case "server-default" => ServerDefault + case "aws-emr" => + val instances = config.getConfigList("instances").asScala.map(c => EMRInstanceRepr.fromConfig(c)) + AWSEMRLaunchData( + launcherSettingsName = config.getString("launcher-settings-name"), + releaseLabel = config.getString("release-label"), + instances = instances + ) + case x => throw new IllegalArgumentException(s"Unknown launch data type $x") + } + } + } + + val ContextConfigRepr: NamedConfigRepr[ContextConfig] = new NamedConfigRepr[ContextConfig] { val allowedTypes = Set( ConfigValueType.STRING, @@ -79,7 +127,8 @@ object ConfigRepr { workerMode = runMode(config.getString("worker-mode")) , runOptions = config.getString("run-options"), streamingDuration = Duration(config.getString("streaming-duration")), - maxConnFailures = config.getOptInt("max-conn-failures").getOrElse(5) + maxConnFailures = config.getOptInt("max-conn-failures").getOrElse(5), + launchData = config.getOptConfig("launch-data").map(LaunchDataConfigRepr.fromConfig).getOrElse(ServerDefault) ) } @@ -100,7 +149,8 @@ object ConfigRepr { "worker-mode" -> fromAnyRef(a.workerMode.name), "run-options" -> fromAnyRef(a.runOptions), "streaming-duration" -> fromDuration(a.streamingDuration), - "max-conn-failures" -> fromAnyRef(a.maxConnFailures) + "max-conn-failures" -> fromAnyRef(a.maxConnFailures), + "launch-data" -> LaunchDataConfigRepr.toConfig(a.launchData).root() ) fromMap(map.asJava).toConfig } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala index 784862629..f593dede7 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/data/FsStorage.scala @@ -2,20 +2,17 @@ package io.hydrosphere.mist.master.data import java.io.File import java.nio.file.{Files, Path} -import java.util.concurrent.Executors import com.typesafe.config.{ConfigFactory, ConfigRenderOptions} -import io.hydrosphere.mist.master.ContextsSettings import io.hydrosphere.mist.master.data.FsStorage._ -import io.hydrosphere.mist.master.models.{ContextConfig, NamedConfig} +import io.hydrosphere.mist.master.models.NamedConfig import io.hydrosphere.mist.utils.{Logger, fs} -import scala.concurrent.{ExecutionContext, Future} import scala.util._ -class FsStorage[A <: NamedConfig]( +class FsStorage[A]( dir: Path, - repr: ConfigRepr[A], + repr: NamedConfigRepr[A], renderOptions: ConfigRenderOptions = DefaultRenderOptions ) extends Logger with RwLock { self => @@ -77,7 +74,7 @@ object FsStorage { .setJson(false) .setFormatted(true) - def create[A <: NamedConfig](path: String, repr: ConfigRepr[A]): FsStorage[A] = + def create[A](path: String, repr: NamedConfigRepr[A]): FsStorage[A] = new FsStorage[A](checkDirectory(path), repr) } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala similarity index 81% rename from mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala rename to mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala index 21f85d047..0692a2936 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnector.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/Cluster.scala @@ -1,31 +1,29 @@ -package io.hydrosphere.mist.master.execution.workers +package io.hydrosphere.mist.master.execution import akka.actor.{ActorRef, ActorRefFactory} +import io.hydrosphere.mist.master.execution.workers._ import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} import io.hydrosphere.mist.utils.akka.WhenTerminated import scala.concurrent.{Future, Promise} -trait WorkerConnector { +trait Cluster { def askConnection(): Future[PerJobConnection] - def warmUp(): Unit - def shutdown(force: Boolean): Future[Unit] def whenTerminated(): Future[Unit] } -object WorkerConnector { +object Cluster { sealed trait Event object Event { final case class AskConnection(resolve: Promise[PerJobConnection]) extends Event final case class Released(conn: WorkerConnection) extends Event final case class Shutdown(force: Boolean) extends Event - case object WarmUp extends Event final case class ConnTerminated(connId: String) extends Event case object GetStatus } @@ -33,11 +31,11 @@ object WorkerConnector { class ActorBasedWorkerConnector( underlying: ActorRef, termination: Future[Unit] - ) extends WorkerConnector { + ) extends Cluster { override def askConnection(): Future[PerJobConnection] = { val promise = Promise[PerJobConnection] - underlying ! WorkerConnector.Event.AskConnection(promise) + underlying ! Cluster.Event.AskConnection(promise) promise.future } @@ -47,10 +45,6 @@ object WorkerConnector { } override def whenTerminated(): Future[Unit] = termination - - override def warmUp(): Unit = underlying ! WorkerConnector.Event.WarmUp - - } def actorBased( @@ -58,7 +52,7 @@ object WorkerConnector { ctx: ContextConfig, runner: WorkerRunner, af: ActorRefFactory - ): WorkerConnector = { + ): Cluster = { val props = ctx.workerMode match { case RunMode.Shared => SharedConnector.props(id, ctx, runner) case RunMode.ExclusiveContext => ExclusiveConnector.props(id, ctx, runner) diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala new file mode 100644 index 000000000..e4610e266 --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClusterRunner.scala @@ -0,0 +1,41 @@ +package io.hydrosphere.mist.master.execution + +import akka.actor.ActorSystem +import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter +import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerRunner} +import io.hydrosphere.mist.master.models.ContextConfig +import io.hydrosphere.mist.utils.akka.ActorRegHub + +import scala.concurrent.Future + +trait ClusterRunner { + + def run(id: String, ctx: ContextConfig): Future[Cluster] + +} + +object ClusterRunner { + + def legacy( + spawn: SpawnSettings, + regHub: ActorRegHub, + workerStarter: WorkerStarter, + system: ActorSystem + ): ClusterRunner = { + new ClusterRunner { + val runner = WorkerRunner.default(spawn, workerStarter, regHub, system) + override def run(id: String, ctx: ContextConfig): Future[Cluster] = { + Future.successful(Cluster.actorBased(id, ctx, runner, system)) + } + } + } + + def awsEmr( + spawn: SpawnSettings, + regHub: ActorRegHub, + system: ActorSystem + ): ClusterRunner = { + ??? + } +} + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala new file mode 100644 index 000000000..0c8be255d --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ClustersService.scala @@ -0,0 +1,53 @@ +package io.hydrosphere.mist.master.execution + +import java.nio.file.Paths + +import akka.actor.ActorSystem +import io.hydrosphere.mist.master.execution.aws.EMRClusterRunner +import io.hydrosphere.mist.master.{AWSEMRLaunchSettings, LauncherSettings} +import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, ServerDefault} +import io.hydrosphere.mist.utils.Logger +import io.hydrosphere.mist.utils.akka.ActorRegHub + +import scala.concurrent.Future + +trait ClustersService { + + def start(id: String, ctx: ContextConfig): Future[Cluster] + +} + +object ClustersService extends Logger{ + + def create( + mistHome: String, + spawn: SpawnSettings, + regHub: ActorRegHub, + launchSettings: Map[String, LauncherSettings], + serverDefault: ClusterRunner, + system: ActorSystem + ): ClustersService = { + + new ClustersService { + + val runners = launchSettings.map({case (name, settings) => { + val runner = settings match { + case aws: AWSEMRLaunchSettings => EMRClusterRunner.create(Paths.get(mistHome), spawn, aws, regHub, system) + } + name -> runner + }}) + + override def start(id: String, ctx: ContextConfig): Future[Cluster] = { + ctx.launchData match { + case ServerDefault => serverDefault.run(id, ctx) + case aws: AWSEMRLaunchData => + runners.get(aws.launcherSettingsName) match { + case Some(runner) => runner.run(id, ctx) + case None => Future.failed(new RuntimeException(s"Unknown settings name ${aws.launcherSettingsName} for ctx ${ctx.name}")) + } + } + } + } + } + +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala index 4be597f24..ecd734498 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextEvent.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master.execution -import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest} +import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest} import io.hydrosphere.mist.master.JobDetails import io.hydrosphere.mist.master.models.ContextConfig diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala index e9ee6db69..4a54e0af9 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ContextFrontend.scala @@ -3,18 +3,18 @@ package io.hydrosphere.mist.master.execution import java.util.UUID import akka.actor.{Actor, ActorLogging, ActorRef, Props, Timers} -import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest} +import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest} import io.hydrosphere.mist.master.Messages.StatusMessages.FailedEvent import io.hydrosphere.mist.master.execution.ContextFrontend.Event.JobDied import io.hydrosphere.mist.master.execution.ContextFrontend.{ConnectorState, FrontendStatus} import io.hydrosphere.mist.master.execution.status.StatusReporter -import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnector} +import io.hydrosphere.mist.master.execution.workers.PerJobConnection import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory, LogService} import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} import io.hydrosphere.mist.utils.akka.{ActorF, ActorFSyntax} import mist.api.data.JsData -import scala.concurrent.Promise +import scala.concurrent.{Future, Promise} import scala.concurrent.duration._ import scala.util.{Failure, Success} @@ -23,27 +23,24 @@ trait FrontendBasics { type State = FrontendState[String, ActorRef] val State = FrontendState - def mkStatus(state: State, conn: Option[ConnectorState]): FrontendStatus = { + def mkStatus(state: State, failures: Int, execId: Option[String]): FrontendStatus = { val jobs = state.queued.map({case (k, _) => k -> ExecStatus.Queued}) ++ state.active.map({case (k, _) => k -> ExecStatus.Started}) - FrontendStatus( - jobs = jobs, - conn.map(_.id), - conn.map(_.failedTimes).getOrElse(0) - ) + FrontendStatus(jobs, failures, execId) } - def mkStatus(state: State): FrontendStatus = mkStatus(state, None) - def mkStatus(state: State, conn: ConnectorState): FrontendStatus = mkStatus(state, Some(conn)) + def mkStatus(state: State): FrontendStatus = mkStatus(state, 0, None) + def mkStatus(state: State, conn: ConnectorState): FrontendStatus = mkStatus(state, conn.failedTimes, Option(conn.id)) } + class ContextFrontend( name: String, reporter: StatusReporter, loggersFactory: JobLoggersFactory, - connectorStarter: (String, ContextConfig) => WorkerConnector, + connectorStarter: (String, ContextConfig) => Future[Cluster], jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)], defaultInactiveTimeout: FiniteDuration ) extends Actor @@ -70,8 +67,7 @@ class ContextFrontend( case req: RunJobRequest => timers.cancel(timerKey) val next = mkJob(req, State.empty, sender()) - val (id, connector) = startConnector(ctx) - becomeWithConnector(ctx, next, ConnectorState.initial(id, connector)) + gotoStartingConnector(ctx, next, 0) case Event.Downtime => log.info(s"Context $name was inactive") @@ -81,8 +77,7 @@ class ContextFrontend( // handle UpdateContext for awaitRequest/initial private def becomeAwaitOrConnected(ctx: ContextConfig, state: State): Unit = { if (ctx.precreated && ctx.workerMode == RunMode.Shared) { - val (id, connector) = startConnector(ctx) - becomeWithConnector(ctx, state, ConnectorState.initial(id, connector)) + gotoStartingConnector(ctx, state, 0) } else { val timerKey = s"$name-await-timeout" timers.startSingleTimer(timerKey, Event.Downtime, defaultInactiveTimeout) @@ -90,6 +85,10 @@ class ContextFrontend( } } + private def shouldGoToEmptyWithTimeout(state: State, ctx: ContextConfig): Boolean = { + state.isEmpty && !ctx.precreated && ctx.downtime.isFinite() + } + // handle currentState changes, starting new jobs if it's possible // or awaiting Downtime timeout private def becomeWithConnector( @@ -105,19 +104,8 @@ class ContextFrontend( } } - def shouldGoToEmptyWithTimeout(): Boolean = { - state.isEmpty && !ctx.precreated && ctx.downtime.isFinite() - } - - if (shouldGoToEmptyWithTimeout()) { - val timerKey = s"$name-wait-downtime" - val timeout = ctx.downtime match { - case f: FiniteDuration => f - case _ => defaultInactiveTimeout - } - timers.startSingleTimer(timerKey, Event.Downtime, timeout) - log.info("Context {} - move to inactive state", name) - context become emptyWithConnector(ctx, connectorState, timerKey) + if (shouldGoToEmptyWithTimeout(state, ctx)) { + gotoEmptyWithConnector(ctx, connectorState) } else { val available = ctx.maxJobs - connectorState.all val need = math.min(state.queued.size - connectorState.asked, available) @@ -147,17 +135,12 @@ class ContextFrontend( def becomeNextConn(next: ConnectorState): Unit = becomeWithConnector(ctx, currentState, next) def becomeNext(c: ConnectorState, s: State): Unit = becomeWithConnector(ctx, s, c) - def becomeSleeping(state: State, conn: ConnectorState, brokenCtx: ContextConfig, error: Throwable): Unit = { - currentState.queued.foreach({case (_, ref) => ref ! JobActor.Event.ContextBroken(error)}) - context become sleepingTilUpdate(state, conn, brokenCtx, error) - } { case Event.Status => sender() ! mkStatus(currentState, connectorState) case ContextEvent.UpdateContext(updCtx) => connectorState.connector.shutdown(false) - val (newId, newConn) = startConnector(updCtx) - becomeWithConnector(updCtx, currentState, ConnectorState.initial(newId, newConn)) + gotoStartingConnector(updCtx, currentState, 0) case req: RunJobRequest => becomeNextState(mkJob(req, currentState, sender())) case CancelJobRequest(id) => becomeNextState(cancelJob(id, currentState, sender())) @@ -180,7 +163,7 @@ class ContextFrontend( val newConnState = connectorState.askFailure if (newConnState.failedTimes >= ctx.maxConnFailures) { connectorState.connector.shutdown(false) - becomeSleeping(currentState, newConnState, ctx, e) + becomeSleeping(currentState, ctx, e, newConnState.failedTimes) } else { becomeNextConn(newConnState) } @@ -197,17 +180,32 @@ class ContextFrontend( log.error(e, "Context {} - connector {} was crushed", name, id) val next = connectorState.connectorFailed if (next.failedTimes >= ctx.maxConnFailures) { - becomeSleeping(currentState, next, ctx, e) + becomeSleeping(currentState, ctx, e, next.failedTimes) } else { - val (newId, newConn) = startConnector(ctx) - becomeWithConnector(ctx, currentState, ConnectorState.initial(newId, newConn).copy(failedTimes = next.failedTimes)) + gotoStartingConnector(ctx, currentState, next.failedTimes) } + //TODO is it possible to stop connector outside except ContextFrontend??? case Event.ConnectorStopped(id) if id == connectorState.id => log.info("Context {} - connector {} was stopped", name, id) - val (newId, newConn) = startConnector(ctx) - becomeWithConnector(ctx, currentState, ConnectorState.initial(newId, newConn)) + gotoStartingConnector(ctx, currentState, 0) + + case Event.ConnectorStarted(_, conn) => conn.shutdown(true) + } + } + + private def gotoEmptyWithConnector( + ctx: ContextConfig, + connectorState: ConnectorState + ): Unit = { + val timerKey = s"$name-wait-downtime" + val timeout = ctx.downtime match { + case f: FiniteDuration => f + case _ => defaultInactiveTimeout } + timers.startSingleTimer(timerKey, Event.Downtime, timeout) + log.info("Context {} - move to inactive state", name) + context become emptyWithConnector(ctx, connectorState, timerKey) } // optional state - use it if ctx isn't precreated @@ -219,8 +217,7 @@ class ContextFrontend( case Event.Status => sender() ! mkStatus(State.empty[String, ActorRef], connectorState) case ContextEvent.UpdateContext(updCtx) => connectorState.connector.shutdown(false) - val (newId, newConn) = startConnector(updCtx) - context become emptyWithConnector(updCtx, ConnectorState.initial(newId, newConn), timerKey) + gotoStartingConnector(updCtx, State.empty, 0) case req: RunJobRequest => timers.cancel(timerKey) @@ -247,39 +244,109 @@ class ContextFrontend( case Event.ConnectorStopped(id) if id == connectorState.id => log.info("Context {} was stopped in empty state - shutdown", name) context stop self + + case Event.ConnectorStarted(_, conn) => conn.shutdown(true) + } + + def becomeSleeping( + state: State, + brokenCtx: ContextConfig, + error: Throwable, + failedTimes: Int + ): Unit = { + state.queued.foreach({case (_, ref) => ref ! JobActor.Event.ContextBroken(error)}) + context become sleepingTilUpdate(state, brokenCtx, error, failedTimes) } - private def sleepingTilUpdate(state: State, conn: ConnectorState, brokenCtx: ContextConfig, error: Throwable): Receive = { - case Event.Status => sender() ! mkStatus(state, conn) + private def sleepingTilUpdate( + state: State, + brokenCtx: ContextConfig, + error: Throwable, + failedTimes: Int + ): Receive = { + case Event.Status => sender() ! mkStatus(state, failedTimes, None) case ContextEvent.UpdateContext(updCtx) => becomeAwaitOrConnected(updCtx, FrontendState.empty) case req: RunJobRequest => respondWithError(brokenCtx, req, sender(), error) case CancelJobRequest(id) => val next = cancelJob(id, state, sender()) - context become sleepingTilUpdate(next, conn, brokenCtx, error) + context become sleepingTilUpdate(next, brokenCtx, error, failedTimes) case Event.Connection(_, connection) => connection.release() case JobActor.Event.Completed(id) => - val (conns, next) = state.getWithState(id) match { - case Some((_, Working)) => conn.connectionReleased -> state.done(id) - case Some((_, Waiting)) => conn -> state.done(id) - case None => conn -> state + val next = state.get(id) match { + case Some(_) => state.done(id) + case None => state } - context become sleepingTilUpdate(next, conns, brokenCtx, error) + context become sleepingTilUpdate(next, brokenCtx, error, failedTimes) + + case Event.ConnectorStarted(_, conn) => conn.shutdown(true) } - private def startConnector(ctx: ContextConfig): (String, WorkerConnector) = { + private def startingConnector( + ctx: ContextConfig, + state: State, + connId: String, + failedTimes: Int + ): Receive = { + case Event.Status => sender() ! mkStatus(state) + + // TODO check if it possible to update current connector/cluster or start new + case ContextEvent.UpdateContext(updCtx) => + log.info(s"Update ctx: $name") + gotoStartingConnector(updCtx, state, 0) + + case req: RunJobRequest => + val next = mkJob(req, state, sender()) + context become startingConnector(ctx, next, connId, failedTimes) + + case CancelJobRequest(id) => cancelJob(id, state, sender()) + + case JobActor.Event.Completed(id) => + val next = state.get(id) match { + case Some(_) => state.done(id) + case None => state + } + context become startingConnector(ctx, next, connId, failedTimes) + + case Event.Connection(_, connection) => connection.release() + + case Event.ConnectorStarted(id, conn) if id == connId => + log.info("Connector started {} id: {}", name, id) + val connState = ConnectorState.initial(id, conn).copy(failedTimes = failedTimes) + conn.whenTerminated().onComplete({ + case Success(_) => self ! Event.ConnectorStopped(id) + case Failure(e) => self ! Event.ConnectorCrushed(id, e) + }) + + if (shouldGoToEmptyWithTimeout(state, ctx)) { + gotoEmptyWithConnector(ctx, connState) + } else { + becomeWithConnector(ctx, state, connState) + } + + + case Event.ConnectorStarted(_, conn) => conn.shutdown(true) + case Event.ConnectorStartFailed(id, e) if id == connId => + log.error(e, "Starting connector {} id: {} failed", name, id) + val nextFailed = failedTimes + 1 + if (nextFailed >= ctx.maxConnFailures) { + becomeSleeping(state, ctx, e, nextFailed) + } else { + gotoStartingConnector(ctx, state, nextFailed) + } + } + + private def gotoStartingConnector(ctx: ContextConfig, state: State, failedTimes: Int): Unit = { val id = ctx.name + "_" + UUID.randomUUID().toString - log.info(s"Starting executor $id for $name") - val connector = connectorStarter(id, ctx) - if (ctx.precreated) connector.warmUp() - connector.whenTerminated().onComplete({ - case Success(_) => self ! Event.ConnectorStopped(id) - case Failure(e) => self ! Event.ConnectorCrushed(id, e) - }) - id -> connector + log.info(s"Starting connector $id for $name") + connectorStarter(id, ctx).onComplete { + case Success(connector) => self ! Event.ConnectorStarted(id, connector) + case Failure(e) => self ! Event.ConnectorStartFailed(id, e) + } + context become startingConnector(ctx, state, id, failedTimes) } private def cancelJob(id: String, state: State, respond: ActorRef): State = { @@ -328,6 +395,8 @@ object ContextFrontend { sealed trait Event object Event { + final case class ConnectorStarted(id: String, connector: Cluster) extends Event + final case class ConnectorStartFailed(id: String, e: Throwable) extends Event final case class ConnectorCrushed(id: String, err: Throwable) extends Event final case class ConnectorStopped(id: String) extends Event @@ -343,16 +412,16 @@ object ContextFrontend { case class FrontendStatus( jobs: Map[String, ExecStatus], - executorId: Option[String], - failures: Int + failures: Int, + executorId: Option[String] ) object FrontendStatus { - val empty: FrontendStatus = FrontendStatus(Map.empty, None, 0) + val empty: FrontendStatus = FrontendStatus(Map.empty, 0, None) } case class ConnectorState( id: String, - connector: WorkerConnector, + connector: Cluster, used: Int, asked: Int, failedTimes: Int @@ -366,7 +435,7 @@ object ContextFrontend { } object ConnectorState { - def initial(id: String, connector: WorkerConnector): ConnectorState = + def initial(id: String, connector: Cluster): ConnectorState = ConnectorState(id, connector, 0, 0, 0) } @@ -374,7 +443,7 @@ object ContextFrontend { name: String, status: StatusReporter, loggersFactory: JobLoggersFactory, - connectorStarter: (String, ContextConfig) => WorkerConnector, + connectorStarter: (String, ContextConfig) => Future[Cluster], jobFactory: ActorF[(ActorRef, RunJobRequest, Promise[JsData], StatusReporter, JobLogger)], defaultInactiveTimeout: FiniteDuration ): Props = Props(classOf[ContextFrontend], name, status, loggersFactory, connectorStarter, jobFactory, defaultInactiveTimeout) @@ -384,6 +453,6 @@ object ContextFrontend { name: String, status: StatusReporter, loggersFactory: JobLoggersFactory, - connectorStarter: (String, ContextConfig) => WorkerConnector + connectorStarter: (String, ContextConfig) => Future[Cluster] ): Props = props(name, status, loggersFactory, connectorStarter, ActorF.props(JobActor.props _), 5 minutes) } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala index dcf48ba42..607279e18 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionInfo.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master.execution -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.master.JobResult import io.hydrosphere.mist.master.models.JobStartResponse import mist.api.data.JsData diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala index a6e817a29..c5ac8127d 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/ExecutionService.scala @@ -5,10 +5,9 @@ import akka.pattern.ask import akka.util.Timeout import cats.data._ import cats.implicits._ -import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, JobParams, RunJobRequest} +import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, JobParams, RunJobRequest} import io.hydrosphere.mist.master.Messages.StatusMessages.InitializedEvent import io.hydrosphere.mist.master.execution.status.StatusReporter -import io.hydrosphere.mist.master.execution.workers.WorkerHub import io.hydrosphere.mist.master.logging.LogService import io.hydrosphere.mist.master.models._ import io.hydrosphere.mist.master.store.JobRepository @@ -22,7 +21,6 @@ import scala.concurrent.Future */ class ExecutionService( contextsMaster: ActorRef, - workersHub: WorkerHub, statusReporter: StatusReporter, repo: JobRepository ) { @@ -42,16 +40,16 @@ class ExecutionService( def getHistory(req: JobDetailsRequest): Future[JobDetailsResponse] = repo.getJobs(req) - def workers(): Seq[WorkerLink] = workersHub.workerConnections().map(_.data) - - def getWorkerLink(workerId: String): Option[WorkerLink] = { - workersHub.workerConnection(workerId).map(_.data) - } +// def workers(): Seq[WorkerLink] = workersHub.workerConnections().map(_.data) +// def getWorkerLink(workerId: String): Option[WorkerLink] = { +// workersHub.workerConnection(workerId).map(_.data) +// } - def stopAllWorkers(): Future[Unit] = workersHub.shutdownAllWorkers() - def stopWorker(id: String): Future[Unit] = workersHub.shutdownWorker(id) +// def stopAllWorkers(): Future[Unit] = workersHub.shutdownAllWorkers() +// +// def stopWorker(id: String): Future[Unit] = workersHub.shutdownWorker(id) def startJob(req: JobStartRequest): Future[ExecutionInfo] = { @@ -110,23 +108,21 @@ class ExecutionService( object ExecutionService { def apply( - spawn: SpawnSettings, + clustersService: ClustersService, system: ActorSystem, streamer: EventsStreamer, repo: JobRepository, logService: LogService ): ExecutionService = { - val hub = WorkerHub(spawn, system) val reporter = StatusReporter.reporter(repo, streamer, logService)(system) - val mkContext = ActorF[ContextConfig]((ctx, af) => { - val props = ContextFrontend.props(ctx.name, reporter, logService, hub.start) + val props = ContextFrontend.props(ctx.name, reporter, logService, clustersService.start) val ref = af.actorOf(props) ref ! ContextEvent.UpdateContext(ctx) ref }) val contextsMaster = system.actorOf(ContextsMaster.props(mkContext)) - new ExecutionService(contextsMaster, hub, reporter, repo) + new ExecutionService(contextsMaster, reporter, repo) } } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala index dbe8818c5..9fa590384 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/JobActor.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution import java.io.{PrintWriter, StringWriter} import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, PoisonPill, Props, SupervisorStrategy, Timers} -import io.hydrosphere.mist.core.CommonData._ +import io.hydrosphere.mist.common.CommonData._ import io.hydrosphere.mist.master.Messages.StatusMessages._ import io.hydrosphere.mist.master.execution.status.StatusReporter import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala index da35b4893..8e75ff7d1 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/SpawnSettings.scala @@ -1,13 +1,11 @@ package io.hydrosphere.mist.master.execution -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo -import io.hydrosphere.mist.master.execution.workers.starter.WorkerStarter +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master.models.ContextConfig import scala.concurrent.duration._ case class SpawnSettings( - runnerCmd: WorkerStarter, timeout: Duration, readyTimeout: FiniteDuration, akkaAddress: String, @@ -19,7 +17,6 @@ case class SpawnSettings( def toWorkerInitInfo(ctx: ContextConfig): WorkerInitInfo = WorkerInitInfo( sparkConf = ctx.sparkConf, - maxJobs = ctx.maxJobsOnNode, downtime = ctx.downtime, streamingDuration = ctx.streamingDuration, logService = this.logAddress, diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala deleted file mode 100644 index 1c4440e18..000000000 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/WorkerLink.scala +++ /dev/null @@ -1,11 +0,0 @@ -package io.hydrosphere.mist.master.execution - -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo - -case class WorkerLink( - name: String, - address: String, - sparkUi: Option[String], - initInfo: WorkerInitInfo -) - diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala new file mode 100644 index 000000000..f41a25329 --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/AgentInstall.scala @@ -0,0 +1,43 @@ +package io.hydrosphere.mist.master.execution.aws + +import java.nio.file.Paths + +case class TransferParams( + agentJar: String, + workerJar: String, + targetDir: String +) + +case class AgentRunParams( + agentId: String, + masterAddress: String, + accessKey: String, + secretKey: String, + region: String, + clusterId: String +) + +object AgentInstall { + + def sshCommands( + transferParams: TransferParams, + agentRunParams: AgentRunParams + ): Seq[SSHCmd] = { + + import transferParams._ + import agentRunParams._ + + val agentRemoteJar = s"$targetDir/mist-agent.jar" + Seq( + SSHCmd.Exec(Seq("mkdir", targetDir)), + SSHCmd.CopyFile(agentJar, s"$targetDir/mist-agent.jar"), + SSHCmd.CopyFile(workerJar, s"$targetDir/mist-worker.jar"), + SSHCmd.Exec(Seq( + "java", "-cp", agentRemoteJar, "io.hydrosphere.mist.agent.ClusterAgent", + masterAddress, agentId, accessKey, secretKey, region, clusterId, + s"1>$targetDir/out.log", s"2>$targetDir/out.log", "&" + )) + ) + } + +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala new file mode 100644 index 000000000..1419655fc --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/CompletableFutureOps.scala @@ -0,0 +1,35 @@ +package io.hydrosphere.mist.master.execution.aws + +import java.util.concurrent.CompletableFuture +import java.util.function.{BiConsumer, BiFunction} + +import cats.effect.IO + +import scala.concurrent.{Future, Promise} + +final class CompletableFutureOps[A](val cf: CompletableFuture[A]) extends AnyVal { + + def toFuture: Future[A] = { + val p = Promise[A] + cf.whenComplete(new BiConsumer[A, Throwable] { + override def accept(res: A, err: Throwable): Unit = { + (Option(res), Option(err)) match { + case (Some(r), None) => p.success(r) + case (_, Some(e)) => + e match { + case ce: java.util.concurrent.CompletionException => p.failure(ce.getCause) + case _ => p.failure(e) + } + case (_, _) => p.failure(new IllegalStateException("CompletableFuture was failed without error information")) + } + }}) + p.future + } + + def toIO: IO[A] = IO.fromFuture(IO(toFuture)) + +} + +object JFutureSyntax { + implicit def cfSyntax[A](cf: CompletableFuture[A]): CompletableFutureOps[A] = new CompletableFutureOps(cf) +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala new file mode 100644 index 000000000..860e346ea --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClient.scala @@ -0,0 +1,252 @@ +package io.hydrosphere.mist.master.execution.aws + +import cats._ +import cats.implicits._ +import cats.effect._ +import io.hydrosphere.mist.master.execution.aws.JFutureSyntax._ +import io.hydrosphere.mist.master.models.EMRInstance +import io.hydrosphere.mist.master.models.EMRInstance.{AutoScaling, Ebs, VolumeType} +import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.emr.EMRAsyncClient +import software.amazon.awssdk.services.emr.model.{Unit => _, _} + +import scala.collection.JavaConverters._ +import scala.concurrent.duration.FiniteDuration + +trait EMRClient[F[_]] { + + def start(settings: EMRRunSettings, instances: Seq[EMRInstance.Instance]): F[EmrInfo] + def status(id: String): F[Option[EmrInfo]] + def stop(id: String): F[Unit] + + def awaitStatus( + id: String, + target: EMRStatus, + sleepTime: FiniteDuration, + triesLeft: Int + )(implicit M: MonadError[F, Throwable], T: Timer[F]): F[EmrInfo] = { + + for { + maybeInfo <- status(id) + next <- maybeInfo match { + case Some(info) if info.status == target => + M.pure(info) + case Some(info) if triesLeft > 0 => + T.sleep(sleepTime).flatMap(_ => awaitStatus(id, target, sleepTime, triesLeft - 1)) + case None => + M.raiseError(new IllegalArgumentException(s"Cluster $id doesn't exists")) + } + } yield next + } +} + +object EMRClient { + + class Default(orig: EMRAsyncClient) extends EMRClient[IO] { + + type JFICBuilder = JobFlowInstancesConfig.Builder + + private def ebsConfiguration(ebs: Ebs): EbsConfiguration = { + + val devices = ebs.volumes.map(volume => { + val volumeType = volume.volumeType match { + case VolumeType.Standard => "standard" + case VolumeType.IO1 => "io1" + case VolumeType.GP2 => "gp2" + } + val spec = VolumeSpecification.builder() + .iops(volume.iops) + .sizeInGB(volume.sizeGB) + .volumeType(volumeType) + .build() + + val count = volume.count.getOrElse(1) + EbsBlockDeviceConfig.builder() + .volumeSpecification(spec) + .volumesPerInstance(count) + .build() + }) + + val optimized = ebs.optimized.getOrElse(true) + EbsConfiguration.builder() + .ebsBlockDeviceConfigs(devices.asJavaCollection) + .ebsOptimized(optimized) + .build() + } + + private def mkAutoScaling(as: AutoScaling): AutoScalingPolicy = { + + def mkRule(rule: EMRInstance.Rule): ScalingRule = { + val adjType = rule.adjustmentType match { + case EMRInstance.AdjustmentType.ChangeInCapacity => AdjustmentType.CHANGE_IN_CAPACITY + case EMRInstance.AdjustmentType.ExactCapacity => AdjustmentType.EXACT_CAPACITY + case EMRInstance.AdjustmentType.PercentChangeInCapacity => AdjustmentType.PERCENT_CHANGE_IN_CAPACITY + } + + val scalingPolicy = SimpleScalingPolicyConfiguration.builder() + .adjustmentType(adjType) + .scalingAdjustment(rule.scalingAdjustment) + .coolDown(rule.coolDown) + .build() + + val action = ScalingAction.builder() + .simpleScalingPolicyConfiguration(scalingPolicy) + .build() + + val cmpOp = rule.trigger.comparisonOperator match { + case EMRInstance.ComparisonOperator.GreaterThanOrEqual => ComparisonOperator.GREATER_THAN_OR_EQUAL + case EMRInstance.ComparisonOperator.GreaterThan => ComparisonOperator.GREATER_THAN + case EMRInstance.ComparisonOperator.LessThan => ComparisonOperator.LESS_THAN + case EMRInstance.ComparisonOperator.LessThanOrEqual => ComparisonOperator.LESS_THAN_OR_EQUAL + } + val statistic = rule.trigger.statistic match { + case EMRInstance.Statistic.SampleCount => Statistic.SAMPLE_COUNT + case EMRInstance.Statistic.Average => Statistic.AVERAGE + case EMRInstance.Statistic.Sum => Statistic.SUM + case EMRInstance.Statistic.Minimum => Statistic.MINIMUM + case EMRInstance.Statistic.Maximum => Statistic.MAXIMUM + } + + val dimensions= rule.trigger.dimensions.map(d => + MetricDimension.builder().key(d.key).value(d.value).build() + ) + + val cloudWatchAlarmDefinition = CloudWatchAlarmDefinition.builder() + .comparisonOperator(cmpOp) + .evaluationPeriods(rule.trigger.evaluationPeriods) + .metricName(rule.trigger.metricName) + .namespace(rule.trigger.namespace) + .period(rule.trigger.period) + .threshold(rule.trigger.threshold) + .statistic(statistic) + .unit(rule.trigger.unit) + .dimensions(dimensions.asJavaCollection) + .build() + + + val trigger = ScalingTrigger.builder() + .cloudWatchAlarmDefinition(cloudWatchAlarmDefinition) + .build() + + ScalingRule.builder() + .name(rule.name) + .description(rule.name) + .action(action) + .trigger(trigger) + .build() + } + + AutoScalingPolicy.builder() + .constraints(ScalingConstraints.builder().maxCapacity(as.max).minCapacity(as.min).build()) + .rules(as.rules.map(mkRule).asJavaCollection) + .build() + } + + private def mkInstanceGroup(instance: EMRInstance.Instance): InstanceGroupConfig = { + val roleType = instance.instanceGroupType match { + case EMRInstance.InstanceGroupType.Core => InstanceRoleType.CORE + case EMRInstance.InstanceGroupType.Master => InstanceRoleType.MASTER + case EMRInstance.InstanceGroupType.Task => InstanceRoleType.TASK + } + val market = instance.market.fold(MarketType.ON_DEMAND)({ + case EMRInstance.Market.OnDemand => MarketType.ON_DEMAND + case EMRInstance.Market.Spot => MarketType.SPOT + }) + + val base = InstanceGroupConfig.builder() + .name(instance.name.getOrElse(s"Mist${instance.instanceGroupType}")) + .instanceRole(roleType) + .instanceType(instance.instanceType) + .instanceCount(instance.instanceCount) + .market(market) + + val withEbs = instance.ebs match { + case Some(ebs) => base.ebsConfiguration(ebsConfiguration(ebs)) + case None => base + } + + val withBidPrice = instance.bidPrice match { + case Some(p) => withEbs.bidPrice(p) + case None => withEbs + } + + val result = instance.autoScaling match { + case Some(as) => withBidPrice.autoScalingPolicy(mkAutoScaling(as)) + case None => withBidPrice + } + + result.build() + } + + private def mkInstancesConfig(builder: JFICBuilder, instances: Seq[EMRInstance.Instance]): JFICBuilder = { + val groups = instances.map(i => mkInstanceGroup(i)) + builder.instanceGroups(groups.asJavaCollection) + } + + override def start(settings: EMRRunSettings, instances: Seq[EMRInstance.Instance]): IO[EmrInfo] = { + import settings._ + + //TODO: configuration! + val sparkApp = Application.builder().name("Spark").build() + + val initial = JobFlowInstancesConfig.builder() + val addInstances = mkInstancesConfig(initial, instances) + val instancesConfig = addInstances + .keepJobFlowAliveWhenNoSteps(true) + .ec2KeyName(keyPair) + .ec2SubnetId(subnetId) + .additionalMasterSecurityGroups(additionalGroup) + .additionalSlaveSecurityGroups(additionalGroup) + .build() + + val request = RunJobFlowRequest.builder() + .name(s"mist-$name") + .releaseLabel(releaseLabel) + .applications(sparkApp) + .jobFlowRole(emrEc2Role) + .serviceRole(emrRole) + .autoScalingRole(autoScalingRole) + .instances(instancesConfig) + .visibleToAllUsers(true) + .build() + + for { + resp <- orig.runJobFlow(request).toIO + maybeSt <- status(resp.jobFlowId()) + out <- maybeSt match { + case Some(info) => IO.pure(info) + case None => IO.raiseError(new RuntimeException(s"Couldn't get infromation about cluster ${resp.jobFlowId()}")) + } + } yield out + } + + override def status(id: String): IO[Option[EmrInfo]] = { + val req = DescribeClusterRequest.builder().clusterId(id).build() + orig.describeCluster(req).toIO + .map(resp => Option(EmrInfo.fromCluster(resp.cluster()))) + .handleErrorWith({ + case _: EMRException => IO.pure(None) + case e => IO.raiseError(e) + }) + } + + override def stop(id: String): IO[Unit] = { + val req = TerminateJobFlowsRequest.builder().jobFlowIds(id).build() + orig.terminateJobFlows(req).toIO.map(_ => ()) + } + } + + def create(accessKey: String, secretKey: String, region: String): EMRClient[IO] = { + val credentials = AwsCredentials.create(accessKey, secretKey) + val provider = StaticCredentialsProvider.create(credentials) + val reg = Region.of(region) + val emrClient = EMRAsyncClient.builder() + .credentialsProvider(provider) + .region(reg) + .build() + + new Default(emrClient) + } +} + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala new file mode 100644 index 000000000..5a106659c --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRClusterRunner.scala @@ -0,0 +1,133 @@ +package io.hydrosphere.mist.master.execution.aws + +import java.nio.file.{Path, Paths} +import java.util.concurrent.Executors + +import akka.actor.ActorSystem +import cats.effect._ +import cats.implicits._ +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.master.AWSEMRLaunchSettings +import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, StopAction, WorkerConnection, WorkerRunner} +import io.hydrosphere.mist.master.execution.workers.starter.{SparkSubmitBuilder, WorkerProcess, WorkerStarter} +import io.hydrosphere.mist.master.execution.{Cluster, ClusterRunner, SpawnSettings} +import io.hydrosphere.mist.master.models.{AWSEMRLaunchData, ContextConfig, EMRInstance} +import io.hydrosphere.mist.utils.Logger +import io.hydrosphere.mist.utils.akka.ActorRegHub + +import scala.concurrent.duration._ +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.{ExecutionContext, Future} +import scala.util.Failure +import IOTimer.Default + +class EMRClusterRunner( + spawn: SpawnSettings, + launchSettings: AWSEMRLaunchSettings, + regHub: ActorRegHub, + system: ActorSystem, + installAgent: (String, String, String) => Unit, + client: EMRClient[IO] +) extends ClusterRunner { + + private def extractData(ctx: ContextConfig): IO[AWSEMRLaunchData] = { + ctx.launchData match { + case data: AWSEMRLaunchData => IO.pure(data) + case other => + val err = new IllegalArgumentException(s"Invalid launch data for AWSCluster ${other.getClass.getSimpleName}") + IO.raiseError(err) + } + } + + private def mkRunSettings(name: String, data: AWSEMRLaunchData, settings: AWSEMRLaunchSettings): EMRRunSettings = { + EMRRunSettings( + name = name, + keyPair = settings.sshKeyPair, + releaseLabel = data.releaseLabel, + subnetId = settings.subnetId, + additionalGroup = settings.additionalGroup, + emrRole = settings.emrRole, + emrEc2Role = settings.emrEc2Role, + autoScalingRole = settings.autoScalingRole + ) + } + + private def startFully(runSettings: EMRRunSettings, instances: Seq[EMRInstance.Instance], client: EMRClient[IO]): IO[EmrInfo] = { + for { + initial <- client.start(runSettings, instances) + await <- client.awaitStatus(initial.id, EMRStatus.Started, 10 seconds, 40) + } yield await + } + + private def mkRunner(launch: AWSEMRLaunchSettings, host: String): WorkerRunner = { + val starter: WorkerStarter = new WorkerStarter { + val builder = new SparkSubmitBuilder(s"/home/${launch.sshUser}/mist-agent", "/usr/lib/spark") + override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = { + val submitCmd = builder.submitWorker(name, initInfo) :+ "&" + Future { + new SSHClient(host, launch.sshUser, launch.sshKeyPath).install(Seq(SSHCmd.Exec(submitCmd))) + } + WorkerProcess.NonLocal + } + + override def stopAction: StopAction = StopAction.Remote + } + + WorkerRunner.default(spawn, starter, regHub, system) + } + + override def run(id: String, ctx: ContextConfig): Future[Cluster] = { + val io = for { + data <- extractData(ctx) + runSettings = mkRunSettings(id, data, launchSettings) + emrInfo <- startFully(runSettings, data.instances, client) + agentId = s"agent-${emrInfo.id}" + _ <- IO[Unit](installAgent(emrInfo.masterPublicDnsName, agentId, emrInfo.id)) + _ <- IO.fromFuture(IO(regHub.waitRef(agentId, 1 minute))) + runner = mkRunner(launchSettings, emrInfo.masterPublicDnsName) + cluster = Cluster.actorBased(id, ctx, runner, system) + _ = cluster.whenTerminated().onComplete(_ => client.stop(emrInfo.id)) + } yield cluster + + io.unsafeToFuture() + } +} + +object EMRClusterRunner extends Logger { + + def create( + jarsDir: Path, + spawn: SpawnSettings, + launchSettings: AWSEMRLaunchSettings, + regHub: ActorRegHub, + system: ActorSystem + ): ClusterRunner = { + import launchSettings._ + val client = EMRClient.create(accessKey, secretKey, region) + + val installAgent = (host: String, agentId: String, awsId: String) => { + val realDir = jarsDir.toAbsolutePath.toRealPath() + val transfer = TransferParams( + realDir.resolve("mist-agent.jar").toString, + realDir.resolve("mist-worker.jar").toString, + s"/home/$sshUser/mist-agent" + ) + val agentRunParams = AgentRunParams( + agentId, spawn.akkaAddress, accessKey, secretKey, region, awsId + ) + val cmds = AgentInstall.sshCommands(transfer, agentRunParams) + new SSHClient(host, sshUser, sshKeyPath).install(cmds) + () + } + + new EMRClusterRunner( + spawn, + launchSettings, + regHub, + system, + installAgent, + client + ) + } +} + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala new file mode 100644 index 000000000..716291c1b --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRRunSettings.scala @@ -0,0 +1,13 @@ +package io.hydrosphere.mist.master.execution.aws + +case class EMRRunSettings( + name: String, + keyPair: String, + releaseLabel: String, + subnetId: String, + additionalGroup: String, + emrRole: String, + emrEc2Role: String, + autoScalingRole: String +) + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala new file mode 100644 index 000000000..5ce183b0d --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EMRStatus.scala @@ -0,0 +1,23 @@ +package io.hydrosphere.mist.master.execution.aws + +import software.amazon.awssdk.services.emr.{model => emodel } + +sealed trait EMRStatus + +object EMRStatus { + + case object Starting extends EMRStatus + case object Started extends EMRStatus + case object Terminated extends EMRStatus + case object Terminating extends EMRStatus + + def fromCluster(cluster: emodel.Cluster): EMRStatus = cluster.status().state() match { + case emodel.ClusterState.STARTING | emodel.ClusterState.BOOTSTRAPPING => Starting + case emodel.ClusterState.WAITING | emodel.ClusterState.RUNNING => Started + case emodel.ClusterState.TERMINATED | emodel.ClusterState.TERMINATED_WITH_ERRORS => Terminated + case emodel.ClusterState.TERMINATING => Terminating + case emodel.ClusterState.UNKNOWN_TO_SDK_VERSION => throw new RuntimeException("Used amazon sdk version is incompatible with aws") + } + +} + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala new file mode 100644 index 000000000..6b1cf6301 --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/EmrInfo.scala @@ -0,0 +1,16 @@ +package io.hydrosphere.mist.master.execution.aws + +import software.amazon.awssdk.services.emr.{model => emodel } + +case class EmrInfo( + id: String, + masterPublicDnsName: String, + status: EMRStatus +) + +object EmrInfo { + + def fromCluster(cluster: emodel.Cluster): EmrInfo = + EmrInfo(cluster.id(), cluster.masterPublicDnsName(), EMRStatus.fromCluster(cluster)) + +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala new file mode 100644 index 000000000..7e8df1de3 --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/IOTimer.scala @@ -0,0 +1,39 @@ +package io.hydrosphere.mist.master.execution.aws + +import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit} + +import cats.effect.{Clock, IO, Timer} + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +class IOTimer( + ec: ExecutionContext, + sc: ScheduledExecutorService +) extends Timer[IO] { + + override val clock: Clock[IO] = new Clock[IO] { + override def realTime(unit: TimeUnit): IO[Long] = + IO(unit.convert(System.currentTimeMillis(), MILLISECONDS)) + + override def monotonic(unit: TimeUnit): IO[Long] = + IO(unit.convert(System.nanoTime(), NANOSECONDS)) + } + + override def sleep(timespan: FiniteDuration): IO[Unit] = + IO.cancelable { cb => + val tick = new Runnable { + def run() = ec.execute(new Runnable { + def run() = cb(Right(())) + }) + } + val f = sc.schedule(tick, timespan.length, timespan.unit) + IO(f.cancel(false)) + } +} + +object IOTimer { + + implicit val Default = new IOTimer(ExecutionContext.global, Executors.newScheduledThreadPool(2)) + +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala new file mode 100644 index 000000000..d4926ea0f --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/aws/SSHClient.scala @@ -0,0 +1,32 @@ +package io.hydrosphere.mist.master.execution.aws + +import com.decodified.scalassh._ +import cats.implicits._ + +import scala.util.Try + +sealed trait SSHCmd +object SSHCmd { + case class CopyFile(from: String, to: String) extends SSHCmd + case class Exec(cmd: Seq[String]) extends SSHCmd +} + +class SSHClient(host: String, user: String, keyPath: String) { + + val cfgProvider = HostConfig( + login = PublicKeyLogin(user, keyPath), + hostName = host, + port = 22, + hostKeyVerifier = HostKeyVerifiers.DontVerify + ) + + def install(cmds: Seq[SSHCmd]): Try[Unit] = { + SSH(host, cfgProvider) { client => + cmds.toList.map { + case SSHCmd.CopyFile(from, to) => client.upload(from, to) + case SSHCmd.Exec(cmd) => client.exec(Command(cmd.mkString(" "))).map(_ => ()) + }.combineAll + }.map(_ => ()) + } +} + diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala index 7008fd1b4..951c3da49 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/status/StatusReporter.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.execution.status import akka.actor.ActorSystem -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.{EventsStreamer, JobDetails} import io.hydrosphere.mist.master.Messages.StatusMessages._ import io.hydrosphere.mist.master.logging.LogService diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala new file mode 100644 index 000000000..ca88142f8 --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/Cancel.scala @@ -0,0 +1,34 @@ +package io.hydrosphere.mist.master.execution.workers + +import akka.actor.{Actor, ActorRef} + +import scala.concurrent.Future + +trait ActorFutureHandler { that: Actor => + + private var realRef: Option[ActorRef] = None + + override def preStart(): Unit = { + that.preStart() + realRef = Some(self) + } + + override def postStop(): Unit = { + that.postStop() + realRef = None + } + + def subscribe[A, B, C](future: Future[A])(f: A => B, g: Throwable => C): Unit = { + future.onComplete(res => { + realRef match { + case None => + case Some(ref) => + val msg = res match { + case scala.util.Success(v) => f(v) + case scala.util.Failure(e) => g(e) + } + ref ! msg + } + })(context.dispatcher) + } +} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala index ca9d4c258..874c9ce50 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnector.scala @@ -2,10 +2,10 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.pattern.pipe -import io.hydrosphere.mist.core.CommonData -import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest} -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest} import io.hydrosphere.mist.master.models.ContextConfig +import io.hydrosphere.mist.master.execution.Cluster import scala.collection.immutable.Queue import scala.concurrent.{Future, Promise} @@ -17,7 +17,7 @@ class ExclusiveConnector( startConnection: (String, ContextConfig) => Future[WorkerConnection] ) extends Actor with ActorLogging { - import WorkerConnector._ + import Cluster._ import context.dispatcher type Conns = Map[String, WorkerConnection] @@ -52,9 +52,6 @@ class ExclusiveConnector( req.failure(e) context become process(other, working, startingConnections - 1) - case Event.WarmUp => - log.warning("Exclusive connector {}: {} received warmup event", id, ctx.name) - case Event.Released(conn) => conn.shutdown(true) context become process(requests, working - conn.id, startingConnections) @@ -105,7 +102,7 @@ object ExclusiveConnector { ref.tell(WorkerBridge.Event.CompleteAndShutdown, ActorRef.noSender) } def cancel(id: String, respond: ActorRef): Unit = ref.tell(CancelJobRequest(id), respond) - def release(): Unit = connector ! Released(direct) + def release(): Unit = connector ! Cluster.Event.Released(direct) } def wrappedConnection(connector: ActorRef, conn: WorkerConnection): PerJobConnection = diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala index 022ae3b58..2796bd5a2 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/PerJobConnection.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.ActorRef -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import scala.concurrent.Future diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala index 60737d391..71bf793ee 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/SharedConnector.scala @@ -4,11 +4,10 @@ import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.pattern.pipe -import io.hydrosphere.mist.core.CommonData -import io.hydrosphere.mist.core.CommonData.CancelJobRequest -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released -import io.hydrosphere.mist.master.models.ContextConfig +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.common.CommonData.CancelJobRequest +import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} +import io.hydrosphere.mist.master.execution.Cluster import scala.collection.immutable.Queue import scala.concurrent.{Future, Promise} @@ -20,6 +19,7 @@ class SharedConnector( idGen: AtomicInteger = new AtomicInteger(1) ) extends Actor with ActorLogging { + import Cluster._ import context.dispatcher private def startConnection(): Future[WorkerConnection] = { @@ -27,6 +27,13 @@ class SharedConnector( connectionStarter(connectionId) } + override def preStart(): Unit = { + if (ctx.precreated) { + (0 until ctx.maxJobs).foreach(_ => startConnection() pipeTo self) + context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobs) + } + } + override def receive: Receive = noConnection private def noConnection: Receive = { @@ -35,10 +42,6 @@ class SharedConnector( case Event.AskConnection(req) => startConnection() pipeTo self context become process(Queue(req), Queue.empty, Map.empty, 1) - - case Event.WarmUp => - (0 until ctx.maxJobsOnNode).foreach(_ => startConnection() pipeTo self) - context become process(Queue.empty, Queue.empty, Map.empty, ctx.maxJobsOnNode) } private def process( @@ -85,7 +88,7 @@ class SharedConnector( context become process(requests, pool, inUse, startingConnections - 1) } - case Event.AskConnection(req) if pool.isEmpty && inUse.size + startingConnections < ctx.maxJobsOnNode => + case Event.AskConnection(req) if pool.isEmpty && inUse.size + startingConnections < ctx.maxJobs => log.info(s"Pool is empty and we are able to start new one connection: inUse size :${inUse.size}") startConnection() pipeTo self context become process(requests :+ req, pool, inUse, startingConnections + 1) @@ -176,7 +179,7 @@ object SharedConnector { import direct.ref def run(req: CommonData.RunJobRequest, respond: ActorRef): Unit = ref.tell(req, respond) def cancel(id: String, respond: ActorRef): Unit = ref.tell(CancelJobRequest(id), respond) - def release(): Unit = connector ! Released(direct) + def release(): Unit = connector ! Cluster.Event.Released(direct) } def wrappedConnection(connector: ActorRef, workerConn: WorkerConnection) = diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala index 977d56034..a7b64d47c 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridge.scala @@ -1,8 +1,8 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.{Actor, ActorLogging, ActorRef, ActorRefFactory, Props, ReceiveTimeout, Terminated, Timers} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.master.execution.WorkerLink +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.master.models.WorkerLink import scala.concurrent.{Future, Promise} import scala.concurrent.duration._ diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala index 7af836694..f18448842 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnection.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.ActorRef -import io.hydrosphere.mist.master.execution.WorkerLink +import io.hydrosphere.mist.master.models.WorkerLink import scala.concurrent.Future diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala index 6c57413c1..cfb1d35fc 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerHub.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.ActorSystem -import io.hydrosphere.mist.master.execution.SpawnSettings +import io.hydrosphere.mist.master.execution.{SpawnSettings, Cluster} import io.hydrosphere.mist.master.models.ContextConfig import io.hydrosphere.mist.utils.akka.ActorRegHub @@ -12,54 +12,54 @@ import scala.util.{Failure, Success} /** * Mix connections mirror and connector starter */ -class WorkerHub( - runner: WorkerRunner, - mkConnector: (String, ContextConfig, WorkerRunner) => WorkerConnector -) extends ConnectionsMirror { - - private val wrappedRunner = new WorkerRunner { - override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = { - val pr = Promise[WorkerConnection] - runner(id, ctx).onComplete { - case Success(conn) => - add(conn) - conn.whenTerminated.onComplete({_ => remove(conn.id)}) - pr.success(conn) - case Failure(e) => pr.failure(e) - } - pr.future - } - - } - - def start(id: String, ctx: ContextConfig): WorkerConnector = mkConnector(id, ctx, wrappedRunner) - - // stopping connection return failed future - // because it stops worker forcibly - // calling stop from here means that user understands it - private def shutdownConn(conn: WorkerConnection): Future[Unit] = - conn.shutdown(true).recover({case _ => ()}) - - def shutdownWorker(id: String): Future[Unit] = workerConnection(id) match { - case Some(connection) => shutdownConn(connection) - case None => Future.failed(new IllegalStateException(s"Unknown worker $id")) - } - - def shutdownAllWorkers(): Future[Unit] = - Future.sequence(workerConnections().map(conn => shutdownConn(conn))).map(_ => ()) - -} - -object WorkerHub { - - def apply(spawn: SpawnSettings, system: ActorSystem): WorkerHub = { - val regHub = ActorRegHub("regHub", system) - val runner = WorkerRunner.default(spawn, regHub, system) - val mkConnector = (id: String, ctx: ContextConfig, runner: WorkerRunner) => { - WorkerConnector.actorBased(id,ctx, runner, system) - } - new WorkerHub(runner, mkConnector) - } -} +//class WorkerHub( +// runner: WorkerRunner, +// mkConnector: (String, ContextConfig, WorkerRunner) => Cluster +//) extends ConnectionsMirror { +// +// private val wrappedRunner = new WorkerRunner { +// override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = { +// val pr = Promise[WorkerConnection] +// runner(id, ctx).onComplete { +// case Success(conn) => +// add(conn) +// conn.whenTerminated.onComplete({_ => remove(conn.id)}) +// pr.success(conn) +// case Failure(e) => pr.failure(e) +// } +// pr.future +// } +// +// } +// +// def start(id: String, ctx: ContextConfig): Cluster = mkConnector(id, ctx, wrappedRunner) +// +// // stopping connection return failed future +// // because it stops worker forcibly +// // calling stop from here means that user understands it +// private def shutdownConn(conn: WorkerConnection): Future[Unit] = +// conn.shutdown(true).recover({case _ => ()}) +// +// def shutdownWorker(id: String): Future[Unit] = workerConnection(id) match { +// case Some(connection) => shutdownConn(connection) +// case None => Future.failed(new IllegalStateException(s"Unknown worker $id")) +// } +// +// def shutdownAllWorkers(): Future[Unit] = +// Future.sequence(workerConnections().map(conn => shutdownConn(conn))).map(_ => ()) +// +//} +// +//object WorkerHub { +// +// def apply(spawn: SpawnSettings, system: ActorSystem): WorkerHub = { +// val regHub = ActorRegHub("regHub", system) +// val runner = WorkerRunner.default(spawn, regHub, system) +// val mkConnector = (id: String, ctx: ContextConfig, runner: WorkerRunner) => { +// Cluster.actorBased(id,ctx, runner, system) +// } +// new WorkerHub(runner, mkConnector) +// } +//} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala index b66367beb..bff0f1d67 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunner.scala @@ -1,9 +1,9 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.{ActorRef, ActorRefFactory} -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master.execution.SpawnSettings -import io.hydrosphere.mist.master.execution.workers.starter.WorkerProcess +import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter} import io.hydrosphere.mist.master.models.ContextConfig import io.hydrosphere.mist.utils.akka.ActorRegHub @@ -18,6 +18,7 @@ object WorkerRunner { class DefaultRunner( spawn: SpawnSettings, + runnerCmd: WorkerStarter, regHub: ActorRegHub, connect: (String, WorkerInitInfo, FiniteDuration, ActorRef, StopAction) => Future[WorkerConnection] ) extends WorkerRunner { @@ -58,11 +59,16 @@ object WorkerRunner { } - def default(spawn: SpawnSettings, regHub: ActorRegHub, af: ActorRefFactory): WorkerRunner = { + def default( + spawn: SpawnSettings, + runnerCmd: WorkerStarter, + regHub: ActorRegHub, + af: ActorRefFactory + ): WorkerRunner = { val connect = (id: String, info: WorkerInitInfo, ready: FiniteDuration, remote: ActorRef, stopAction: StopAction) => { WorkerBridge.connect(id, info, ready, remote, stopAction)(af) } - new DefaultRunner(spawn, regHub, connect) + new DefaultRunner(spawn, runnerCmd, regHub, connect) } } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala index 78177776d..d763c80fd 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/DockerStarter.scala @@ -6,7 +6,7 @@ import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.command.{CreateContainerCmd, InspectContainerResponse} import com.github.dockerjava.api.model.{ContainerNetwork, Link} import com.github.dockerjava.core.{DefaultDockerClientConfig, DockerClientBuilder} -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master._ import io.hydrosphere.mist.master.execution.workers.StopAction import io.hydrosphere.mist.utils.Logger diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala index ecce964ca..f682b75b8 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/LocalSparkSubmit.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter import java.nio.file.Path -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master.execution.workers.StopAction import io.hydrosphere.mist.utils.Logger diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala index 6c1d14f56..c24477d6c 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/ManualStarter.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter import java.nio.file.Path -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master.ManualRunnerConfig import io.hydrosphere.mist.master.execution.workers.StopAction import io.hydrosphere.mist.utils.Logger diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala index e0c1c08cb..79984fd8b 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilder.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter import java.nio.file.Paths -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo class SparkSubmitBuilder(mistHome: String, sparkHome: String) extends PsUtil { diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala index 5d8b01336..a1ec5d773 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/execution/workers/starter/WorkerStarter.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.execution.workers.starter import java.nio.file.Path -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master._ import io.hydrosphere.mist.master.execution.workers.StopAction diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala index 03edd9b3a..8cb0ce385 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/HttpV2Routes.scala @@ -157,33 +157,33 @@ object HttpV2Routes extends Logger { import HttpV2Base._ - def workerRoutes(jobService: ExecutionService): Route = { - path( root / "workers" ) { - get { complete(jobService.workers()) } - } ~ - path( root / "workers" / Segment ) { workerId => - delete { - completeU(jobService.stopWorker(workerId)) - } ~ - get { - completeOpt { jobService.getWorkerLink(workerId) } - } - } ~ - path( root / "workers"/ Segment / "jobs") { workerId => - get { (paginationQuery & statusesQuery) { (pagination, statuses) => - val req = JobDetailsRequest(pagination.limit, pagination.offset) - .withFilter(FilterClause.ByStatuses(statuses)) - .withFilter(FilterClause.ByWorkerId(workerId)) - - onSuccess(jobService.getHistory(req))(rsp => { - if (pagination.paginate) - complete(rsp) - else - complete(rsp.jobs) - }) - } - }} - } +// def workerRoutes(jobService: ExecutionService): Route = { +// path( root / "workers" ) { +// get { complete(jobService.workers()) } +// } ~ +// path( root / "workers" / Segment ) { workerId => +// delete { +// completeU(jobService.stopWorker(workerId)) +// } ~ +// get { +// completeOpt { jobService.getWorkerLink(workerId) } +// } +// } ~ +// path( root / "workers"/ Segment / "jobs") { workerId => +// get { (paginationQuery & statusesQuery) { (pagination, statuses) => +// val req = JobDetailsRequest(pagination.limit, pagination.offset) +// .withFilter(FilterClause.ByStatuses(statuses)) +// .withFilter(FilterClause.ByWorkerId(workerId)) +// +// onSuccess(jobService.getHistory(req))(rsp => { +// if (pagination.paginate) +// complete(rsp) +// else +// complete(rsp.jobs) +// }) +// } +// }} +// } def functionsCrud(functions: FunctionsService): Route = { path( root / "functions" ) { @@ -442,7 +442,7 @@ object HttpV2Routes extends Logger { handleExceptions(exceptionHandler) { functionAllRoutes(main) ~ jobsRoutes(main) ~ - workerRoutes(main.execution) ~ +// workerRoutes(main.execution) ~ contextsRoutes(main) ~ internalArtifacts(mistHome) ~ artifactRoutes(artifacts) ~ diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala index 36bd3adbf..4709bd93b 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/http/models.scala @@ -3,8 +3,8 @@ package io.hydrosphere.mist.master.interfaces.http import java.lang.management._ import java.time.LocalDateTime -import io.hydrosphere.mist.core.FunctionInfoData -import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} +import io.hydrosphere.mist.common.FunctionInfoData +import io.hydrosphere.mist.master.models.{ContextConfig, LaunchData, RunMode} import mist.api._ import scala.concurrent.duration.Duration @@ -122,7 +122,8 @@ case class ContextCreateRequest( workerMode: Option[RunMode] = None, runOptions: Option[String] = None, streamingDuration: Option[Duration] = None, - maxConnFailures: Option[Int] = None + maxConnFailures: Option[Int] = None, + launchData: Option[LaunchData] = None ) { def toContextWithFallback(other: ContextConfig): ContextConfig = @@ -135,7 +136,8 @@ case class ContextCreateRequest( runOptions.getOrElse(other.runOptions), workerMode.getOrElse(other.workerMode), streamingDuration.getOrElse(other.streamingDuration), - maxConnFailures.getOrElse(other.maxConnFailures) + maxConnFailures.getOrElse(other.maxConnFailures), + launchData.getOrElse(other.launchData) ) } diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala index 0231f49df..aaf2e234b 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/interfaces/jsonCodecs.scala @@ -3,12 +3,11 @@ package io.hydrosphere.mist.master.interfaces import java.time.LocalDateTime import java.time.format.{DateTimeFormatter, DateTimeParseException} -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, WorkerInitInfo} -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, WorkerInitInfo} +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.Messages.StatusMessages._ -import io.hydrosphere.mist.master.execution.WorkerLink import io.hydrosphere.mist.master.interfaces.http._ -import io.hydrosphere.mist.master.models._ +import io.hydrosphere.mist.master.models.{WorkerLink, _} import io.hydrosphere.mist.master.{JobDetails, JobDetailsResponse, JobResult} import mist.api.{data => mdata} import spray.json._ @@ -16,6 +15,7 @@ import spray.json._ import scala.collection.JavaConversions._ import scala.concurrent.duration._ import scala.util.Try +import scala.util.parsing.json.JSONObject trait AnyJsonFormat extends DefaultJsonProtocol { @@ -125,10 +125,94 @@ trait JobDetailsJsonFormat extends DefaultJsonProtocol with AnyJsonFormat with M } +trait EMRInstanceFormat extends DefaultJsonProtocol { + + def dummyAdtStringFormat[A](hint: String, pairs: Seq[(A, String)]): JsonFormat[A] = new JsonFormat[A] { + + override def write(obj: A): JsValue = pairs.find(a => a._1 == obj) match { + case Some((_, v)) => JsString(v) + case None => throw new IllegalArgumentException(s"Undeclared object $obj in $hint format") + } + override def read(json: JsValue): A = json match { + case JsString(v) => pairs.find(a => a._2 == v) match { + case Some(out) => out._1 + case None => throw new IllegalArgumentException(s"Undeclared string value $v in $hint format") + } + case _ => throw new IllegalArgumentException(s"Invalid input value for $hint format") + } + } + + implicit val marketF = dummyAdtStringFormat[EMRInstance.Market]( + hint = "Market", + pairs = Seq( + EMRInstance.Market.OnDemand -> "onDemand", + EMRInstance.Market.Spot -> "spot" + ) + ) + + implicit val instanceGroupTYpeF = dummyAdtStringFormat[EMRInstance.InstanceGroupType]( + hint = "InstanceGroupType", + pairs = Seq( + EMRInstance.InstanceGroupType.Master -> "master", + EMRInstance.InstanceGroupType.Core -> "core", + EMRInstance.InstanceGroupType.Task -> "task" + ) + ) + + implicit val volumeTypeF = dummyAdtStringFormat[EMRInstance.VolumeType]( + hint = "VolumeType", + pairs = Seq( + EMRInstance.VolumeType.IO1 -> "io1", + EMRInstance.VolumeType.GP2 -> "gp2", + EMRInstance.VolumeType.Standard -> "standard" + ) + ) + + implicit val ebsVolumeF = jsonFormat4(EMRInstance.EbsVolume.apply) + implicit val ebsF = jsonFormat2(EMRInstance.Ebs.apply) + + implicit val adjustmentTypeF = dummyAdtStringFormat[EMRInstance.AdjustmentType]( + hint = "AdjustmentType", + pairs = Seq( + EMRInstance.AdjustmentType.ChangeInCapacity -> "changeInCapacity", + EMRInstance.AdjustmentType.PercentChangeInCapacity -> "percentChangeInCapacity", + EMRInstance.AdjustmentType.ExactCapacity -> "exactCapacity" + ) + ) + implicit val cmpOperatorF = dummyAdtStringFormat[EMRInstance.ComparisonOperator]( + hint = "ComparisonOperator", + pairs = Seq( + EMRInstance.ComparisonOperator.GreaterThanOrEqual -> "greaterThanOrEqual", + EMRInstance.ComparisonOperator.GreaterThan -> "greaterThan", + EMRInstance.ComparisonOperator.LessThan -> "lessThan", + EMRInstance.ComparisonOperator.LessThanOrEqual -> "lessThanOrEqual" + ) + ) + implicit val statisticF = dummyAdtStringFormat[EMRInstance.Statistic]( + hint = "Statistic", + pairs = Seq( + EMRInstance.Statistic.SampleCount -> "sampleCount", + EMRInstance.Statistic.Average -> "average", + EMRInstance.Statistic.Sum -> "sum", + EMRInstance.Statistic.Minimum -> "minimum", + EMRInstance.Statistic.Maximum -> "maximum" + ) + ) + implicit val dimensionsF = jsonFormat2(EMRInstance.Dimension.apply) + implicit val triggerF = jsonFormat9(EMRInstance.Trigger.apply) + implicit val ruleF = jsonFormat6(EMRInstance.Rule.apply) + + implicit val autoScalingF = jsonFormat3(EMRInstance.AutoScaling.apply) + implicit val instanceF = jsonFormat8(EMRInstance.Instance.apply) + +} + +object EMRInstanceFormat extends EMRInstanceFormat trait JsonCodecs extends SprayJsonSupport with DefaultJsonProtocol with AnyJsonFormat + with EMRInstanceFormat with JobDetailsJsonFormat { implicit val printer = CompactPrinter @@ -167,7 +251,7 @@ trait JsonCodecs extends SprayJsonSupport implicit val workerInitInfoF = rootFormat(lazyFormat(jsonFormat(WorkerInitInfo.apply, - "sparkConf", "maxJobs", "downtime", "streamingDuration", "logService", "masterAddress","masterHttpConf", "maxArtifactSize", "runOptions"))) + "sparkConf", "downtime", "streamingDuration", "logService", "masterAddress","masterHttpConf", "maxArtifactSize", "runOptions"))) implicit val workerLinkF = rootFormat(lazyFormat(jsonFormat(WorkerLink.apply, @@ -227,10 +311,37 @@ trait JsonCodecs extends SprayJsonSupport implicit val devJobStartReqModelF = jsonFormat7(DevJobStartRequestModel.apply) + implicit val emrInstancesF = EMRInstanceFormat - implicit val contextConfigF = jsonFormat9(ContextConfig.apply) + implicit val launchDataF = new JsonFormat[LaunchData] { + + val ServerDefaultKey = "server-default" + val AWSEMRKey = "aws-emr" + + val awsEmrLaunchDataF = jsonFormat3(AWSEMRLaunchData.apply) + + override def write(in: LaunchData): JsValue = in match { + case ServerDefault => JsObject("type" -> JsString(ServerDefaultKey)) + case aws: AWSEMRLaunchData => + val data = awsEmrLaunchDataF.write(aws).asJsObject.fields + JsObject(data + ("type" -> JsString(AWSEMRKey))) + } + + override def read(json: JsValue): LaunchData = { + def fromType(t: String, obj: JsValue): LaunchData = t match { + case `ServerDefaultKey` => ServerDefault + case `AWSEMRKey` => awsEmrLaunchDataF.read(obj) + } + val t = json.asJsObject.fields.getOrElse("type", JsNull) + t match { + case JsString(v) => fromType(v, json) + case x => throw new IllegalArgumentException(s"Invalid launch data format $x") + } + } + } + implicit val contextConfigF = jsonFormat10(ContextConfig.apply) - implicit val contextCreateRequestF = jsonFormat9(ContextCreateRequest.apply) + implicit val contextCreateRequestF = jsonFormat10(ContextCreateRequest.apply) implicit val jobDetailsResponseF = jsonFormat2(JobDetailsResponse.apply) implicit val updateEventF = new JsonFormat[SystemEvent] { diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala index b82ac54f7..53eff7d7b 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionInfoProviderRunner.scala @@ -1,8 +1,8 @@ package io.hydrosphere.mist.master.jobs import akka.actor.{Actor, ActorRef, ActorSystem, Props, ReceiveTimeout} -import io.hydrosphere.mist.core.CommonData -import io.hydrosphere.mist.core.CommonData.RegisterJobInfoProvider +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.common.CommonData.RegisterJobInfoProvider import io.hydrosphere.mist.master.FunctionInfoProviderConfig import scala.concurrent.duration.{Duration, FiniteDuration} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala index 96cc5822c..f74273de8 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/jobs/FunctionsService.scala @@ -7,8 +7,8 @@ import akka.pattern._ import akka.util.Timeout import cats.data._ import cats.implicits._ -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData, PythonEntrySettings} +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData, PythonEntrySettings} import io.hydrosphere.mist.master.artifact.ArtifactRepository import io.hydrosphere.mist.master.data.{Contexts, ContextsStorage, FunctionConfigStorage} import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala index cbaf0e37e..7d00b8e92 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/JobLogger.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.logging import akka.actor.ActorRef -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent trait JobLogger { protected val jobId: String diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala index af1e2b508..1eedf2bca 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/LogStreams.scala @@ -8,7 +8,7 @@ import akka.stream.{ActorAttributes, ActorMaterializer, OverflowStrategy} import akka.util.ByteString import akka.{Done, NotUsed} import com.twitter.chill.{KryoPool, ScalaKryoInstantiator} -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.Messages.StatusMessages.ReceivedLogs import io.hydrosphere.mist.master.{EventsStreamer, LogStoragePaths} import io.hydrosphere.mist.utils.Logger diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala index e445ebd72..884728070 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/logging/writers.scala @@ -5,7 +5,7 @@ import java.nio.file._ import akka.actor._ import akka.pattern.ask import akka.util.Timeout -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.LogStoragePaths import scala.concurrent.Future diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala index 562fcf216..3c4a90157 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/base.scala @@ -30,7 +30,115 @@ trait NamedConfig { val name: String } -case class ContextConfig( +sealed trait LaunchData +/** use default worker-runner **/ +case object ServerDefault extends LaunchData + +object EMRInstance { + + sealed trait InstanceGroupType + object InstanceGroupType { + case object Master extends InstanceGroupType + case object Core extends InstanceGroupType + case object Task extends InstanceGroupType + } + + sealed trait Market + object Market { + case object OnDemand extends Market + case object Spot extends Market + } + + sealed trait VolumeType + object VolumeType { + case object Standard extends VolumeType + case object IO1 extends VolumeType + case object GP2 extends VolumeType + } + final case class EbsVolume( + volumeType: VolumeType, + sizeGB: Int, + iops: Int, + count: Option[Int] + ) + + final case class Ebs( + optimized: Option[Boolean], + volumes: Seq[EbsVolume] + ) + + sealed trait AdjustmentType + object AdjustmentType { + case object ChangeInCapacity extends AdjustmentType + case object PercentChangeInCapacity extends AdjustmentType + case object ExactCapacity extends AdjustmentType + } + + sealed trait ComparisonOperator + object ComparisonOperator { + case object GreaterThanOrEqual extends ComparisonOperator + case object GreaterThan extends ComparisonOperator + case object LessThan extends ComparisonOperator + case object LessThanOrEqual extends ComparisonOperator + } + sealed trait Statistic + object Statistic { + case object SampleCount extends Statistic + case object Average extends Statistic + case object Sum extends Statistic + case object Minimum extends Statistic + case object Maximum extends Statistic + } + + final case class Dimension(key:String, value: String) + + final case class Trigger( + comparisonOperator: ComparisonOperator, + evaluationPeriods: Int, + metricName: String, + namespace: String, + period: Int, + threshold: Double, + statistic: Statistic, + unit: String, + dimensions: Seq[Dimension] + ) + + final case class Rule( + name: String, + description: String, + adjustmentType: AdjustmentType, + scalingAdjustment: Int, + coolDown: Int, + trigger: Trigger + ) + + final case class AutoScaling( + max: Int, + min: Int, + rules: Seq[Rule] + ) + + final case class Instance( + instanceType: String, + instanceGroupType: InstanceGroupType, + name: Option[String], + instanceCount: Int, + market: Option[Market], + ebs: Option[Ebs], + bidPrice: Option[String], + autoScaling: Option[AutoScaling] + ) + +} + +final case class AWSEMRLaunchData( + launcherSettingsName: String, + releaseLabel: String, + instances: Seq[EMRInstance.Instance] +) extends LaunchData + +final case class ContextConfig( name: String, sparkConf: Map[String, String], downtime: Duration, @@ -39,17 +147,12 @@ case class ContextConfig( runOptions: String, workerMode: RunMode, streamingDuration: Duration, - maxConnFailures: Int -) extends NamedConfig { - - def maxJobsOnNode: Int = workerMode match { - case RunMode.Shared => maxJobs - case RunMode.ExclusiveContext => 1 - } + maxConnFailures: Int, + launchData: LaunchData +) extends NamedConfig -} -case class FunctionConfig( +final case class FunctionConfig( name: String, path: String, className: String, diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala new file mode 100644 index 000000000..475d4c28d --- /dev/null +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/execution.scala @@ -0,0 +1,16 @@ +package io.hydrosphere.mist.master.models + +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo + +case class WorkerLink( + name: String, + address: String, + sparkUi: Option[String], + initInfo: WorkerInitInfo +) + +case class ClusterLink( + name: String, + info: String, + workers: Seq[WorkerLink] +) \ No newline at end of file diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala index 1cb7573e2..46170bda1 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/models/models.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.models import java.util.UUID -import io.hydrosphere.mist.core.CommonData.Action -import io.hydrosphere.mist.core.FunctionInfoData +import io.hydrosphere.mist.common.CommonData.Action +import io.hydrosphere.mist.common.FunctionInfoData import io.hydrosphere.mist.master.JobDetails import mist.api.data.{JsData, JsMap} diff --git a/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala b/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala index 0f93ace23..e1145e5c8 100644 --- a/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala +++ b/mist/master/src/main/scala/io/hydrosphere/mist/master/store/H2JobsRepository.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.store import java.nio.file.Paths -import io.hydrosphere.mist.core.CommonData.{Action, JobParams} +import io.hydrosphere.mist.common.CommonData.{Action, JobParams} import io.hydrosphere.mist.master.{FilterClause, JobDetails, JobDetailsRequest, JobDetailsResponse} import io.hydrosphere.mist.master.interfaces.JsonCodecs import JsonCodecs._ diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala index fc55dbf59..2c63418af 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/ContextsCrudLikeSpec.scala @@ -1,10 +1,10 @@ package io.hydrosphere.mist.master -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.data.ContextsStorage import io.hydrosphere.mist.master.execution.ExecutionService import io.hydrosphere.mist.master.interfaces.http.ContextCreateRequest -import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} +import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault} import org.scalatest.{FunSpec, Matchers} import org.mockito.Mockito._ @@ -17,7 +17,7 @@ class ContextsCrudLikeSpec extends FunSpec with Matchers with TestData with Mock val storage = mock[ContextsStorage] val executionS = mock[ExecutionService] - val defaultValue = ContextConfig("default", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.Shared, 1 seconds, 5) + val defaultValue = ContextConfig("default", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.Shared, 1 seconds, 5, ServerDefault) val req = ContextCreateRequest("yoyo", workerMode = Some(RunMode.ExclusiveContext)) when(storage.defaultConfig).thenReturn(defaultValue) @@ -29,7 +29,7 @@ class ContextsCrudLikeSpec extends FunSpec with Matchers with TestData with Mock } val result = Await.result(crud.create(req), Duration.Inf) - result shouldBe ContextConfig("yoyo", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.ExclusiveContext, 1 seconds, 5) + result shouldBe ContextConfig("yoyo", Map.empty, Duration.Inf, 20, precreated = false, "", RunMode.ExclusiveContext, 1 seconds, 5, ServerDefault) verify(executionS).updateContext(any[ContextConfig]) } diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala index 26fde4d19..ec9760ccb 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/MainServiceSpec.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.master import akka.actor.ActorSystem import akka.testkit.TestKit -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest} -import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar} +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest} +import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar} import io.hydrosphere.mist.master.artifact.ArtifactRepository import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage} import io.hydrosphere.mist.master.execution.{ExecutionInfo, ExecutionService} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala index 81c3cc3c8..69bac4ff7 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/MistConfigSpec.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master import cats.{Eval, Now} import com.typesafe.config.ConfigFactory -import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} +import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault} import org.scalatest.{FunSpec, Matchers} import scala.concurrent.duration._ @@ -43,7 +43,8 @@ class MistConfigSpec extends FunSpec with Matchers { runOptions = "--opt", workerMode = RunMode.Shared, streamingDuration = 1.seconds, - maxConnFailures = 5 + maxConnFailures = 5, + launchData = ServerDefault )) } diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala index f4f7ead3e..fc9444632 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/TestData.scala @@ -1,9 +1,9 @@ package io.hydrosphere.mist.master import com.typesafe.config.ConfigFactory -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest, WorkerInitInfo} -import io.hydrosphere.mist.core.FunctionInfoData -import io.hydrosphere.mist.master.execution.WorkerLink +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, WorkerInitInfo} +import io.hydrosphere.mist.common.FunctionInfoData +import io.hydrosphere.mist.master.models.WorkerLink import mist.api.ArgInfo import mist.api.data._ import mist.api.encoding.defaultEncoders._ @@ -70,7 +70,6 @@ trait TestData { val workerInitData = WorkerInitInfo( sparkConf = FooContext.sparkConf, - maxJobs = FooContext.maxJobs, downtime = FooContext.downtime, streamingDuration = FooContext.streamingDuration, logService = "localhost:2005", diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala index cc8547ea1..2ffbd4064 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/artifact/ArtifactRepositorySpec.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.artifact import java.io.File import java.nio.file.{Files, Paths} -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.TestUtils.AwaitSyntax import io.hydrosphere.mist.master.data.FunctionConfigStorage import org.apache.commons.io.FileUtils diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala index d75fbdb90..0de263dab 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ConfigReprsSpec.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.data import com.typesafe.config.{ConfigFactory, ConfigObject, ConfigParseOptions, ConfigRenderOptions} -import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, RunMode} +import io.hydrosphere.mist.master.models.{ContextConfig, FunctionConfig, RunMode, ServerDefault} import org.scalatest.{FunSpec, Matchers} import scala.concurrent.duration._ @@ -20,13 +20,13 @@ class ConfigReprsSpec extends FunSpec with Matchers { | namespace = "namespace" """.stripMargin) - val parsed = cfg.to[FunctionConfig]("name") + val parsed = ConfigRepr.EndpointsRepr.fromConfig("name", cfg) parsed shouldBe FunctionConfig("name", "jar_path.jar", "MyJob1", "namespace") } it("should render to raw") { val e = FunctionConfig("name", "jar_path.jar", "MyJob1", "namespace") - val raw = e.toConfig + val raw = ConfigRepr.EndpointsRepr.toConfig(e) raw.getString("path") shouldBe "jar_path.jar" raw.getString("className") shouldBe "MyJob1" @@ -51,12 +51,11 @@ class ConfigReprsSpec extends FunSpec with Matchers { | streaming-duration = 30 seconds | worker-mode = "shared" | max-conn-failures = 10 - | |} """.stripMargin , ConfigParseOptions.defaults()) - val context = cfg.to[ContextConfig]("test") + val context = ConfigRepr.ContextConfigRepr.fromConfig("test", cfg) context.sparkConf shouldBe Map("x.y" -> "z", "a" -> "1") context.downtime shouldBe Duration.Inf context.maxJobs shouldBe 100 @@ -86,7 +85,7 @@ class ConfigReprsSpec extends FunSpec with Matchers { """.stripMargin , ConfigParseOptions.defaults()) - val context = cfg.to[ContextConfig]("test") + val context = ConfigRepr.ContextConfigRepr.fromConfig("test", cfg) context.maxConnFailures shouldBe 5 } @@ -107,7 +106,8 @@ class ConfigReprsSpec extends FunSpec with Matchers { runOptions = "", workerMode = RunMode.Shared, streamingDuration = 1.minutes, - maxConnFailures = 5 + maxConnFailures = 5, + launchData = ServerDefault ) val raw = ConfigRepr.ContextConfigRepr.toConfig(e) Duration(raw.getString("downtime")) shouldBe 10.minutes diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala index e84291d50..80653c88a 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/ContextsStorageSpec.scala @@ -5,7 +5,7 @@ import java.nio.file.Paths import io.hydrosphere.mist.master import io.hydrosphere.mist.master.TestUtils -import io.hydrosphere.mist.master.models.{ContextConfig, RunMode} +import io.hydrosphere.mist.master.models.{ContextConfig, RunMode, ServerDefault} import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfter, FunSpec, Matchers} @@ -38,7 +38,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter { it("should update") { val contexts = testStorage() - val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5) + val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5, ServerDefault) contexts.update(ctx).await contexts.get("new").await.isDefined shouldBe true } @@ -46,7 +46,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter { it("should return defaults") { val contexts = testStorage() - val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5) + val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, false, "weq", RunMode.Shared, 10 second, 5, ServerDefault) contexts.update(ctx).await contexts.all.await.map(_.name) should contain allOf ("default", "foo", "new") @@ -60,7 +60,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter { it("should return precreated") { val contexts = testStorage() - val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, true, "weq", RunMode.Shared, 10 second, 5) + val ctx = ContextConfig("new", Map.empty, Duration.Inf, 50, true, "weq", RunMode.Shared, 10 second, 5, ServerDefault) contexts.update(ctx).await contexts.precreated.await should contain only(ctx) @@ -74,7 +74,7 @@ class ContextsStorageSpec extends FunSpec with Matchers with BeforeAndAfter { it("should override settings") { val contexts = testStorage() - val ctx = ContextConfig("foo", Map.empty, Duration.Inf, 50, true, "FOOOOPT", RunMode.Shared, 10 second, 5) + val ctx = ContextConfig("foo", Map.empty, Duration.Inf, 50, true, "FOOOOPT", RunMode.Shared, 10 second, 5, ServerDefault) contexts.get("foo").await.get.runOptions shouldNot be (ctx.runOptions) contexts.update(ctx).await diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala index 41e81ae66..0f3ba316e 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/data/FStorageSpec.scala @@ -14,7 +14,7 @@ class FStorageSpec extends FunSpec with Matchers with BeforeAndAfter { value: Int ) extends NamedConfig - val testEntryConfigRepr = new ConfigRepr[TestEntry] { + val testEntryConfigRepr = new NamedConfigRepr[TestEntry] { import scala.collection.JavaConverters._ override def fromConfig(config: Config): TestEntry = { diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala index 44775e836..60a5aeb0a 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextFrontendSpec.scala @@ -1,12 +1,15 @@ package io.hydrosphere.mist.master.execution +import java.util.concurrent.atomic.AtomicInteger + import akka.actor.ActorRef import akka.testkit.{TestActorRef, TestProbe} -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest, _} -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest, _} +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.execution.status.StatusReporter -import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection, WorkerConnector} +import io.hydrosphere.mist.master.execution.workers.PerJobConnection import io.hydrosphere.mist.master.logging.{JobLogger, JobLoggersFactory} +import io.hydrosphere.mist.master.models.ContextConfig import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData, TestUtils} import io.hydrosphere.mist.utils.akka.ActorF import mist.api.data.JsMap @@ -35,7 +38,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -62,7 +65,8 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") probe.send(frontend, ContextFrontend.Event.Status) val status2 = probe.expectMsgType[ContextFrontend.FrontendStatus] status2.executorId.isDefined shouldBe true - status2.jobs should contain only("id" -> ExecStatus.Started) + status2.jobs.size shouldBe 1 + status2.jobs.head shouldBe "id" -> ExecStatus.Started job.send(frontend, JobActor.Event.Completed("id")) @@ -72,27 +76,6 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") status3.jobs.isEmpty shouldBe true } - it("should warmup precreated") { - val connector = mock[WorkerConnector] - when(connector.whenTerminated).thenReturn(Promise[Unit].future) - - val job = mkJobProbe() - val props = ContextFrontend.props( - name = "name", - status = StatusReporter.NOOP, - loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, - jobFactory = ActorF.static(job.ref), - defaultInactiveTimeout = 5 minutes - ) - val frontend = TestActorRef[ContextFrontend](props) - frontend ! ContextEvent.UpdateContext(TestUtils.FooContext.copy(precreated = true)) - - eventually(timeout(Span(3, Seconds))) { - verify(connector).warmUp() - } - } - it("should respect idle timeout - awaitRequest") { val connector = successfulConnector() @@ -101,7 +84,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 1 second ) @@ -119,7 +102,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 1 second ) @@ -148,7 +131,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -175,13 +158,19 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") } it("should restart connector 'til max start times and then sleep") { + val callCounter = new AtomicInteger(0) val connector = crushedConnector() + val starter = (id: String, ctx: ContextConfig) => { + callCounter.incrementAndGet() + Future.successful(connector) + } + val job = mkJobProbe() val props = ContextFrontend.props( name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = starter, jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -192,11 +181,13 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") probe.send(frontend, RunJobRequest(s"id", JobParams("path", "MyClass", JsMap.empty, Action.Execute))) probe.expectMsgType[ExecutionInfo] + job.expectMsgType[JobActor.Event.ContextBroken] + probe.send(frontend, ContextFrontend.Event.Status) val status = probe.expectMsgType[ContextFrontend.FrontendStatus] status.failures shouldBe TestUtils.FooContext.maxConnFailures + callCounter.get() shouldBe TestUtils.FooContext.maxConnFailures - job.expectMsgType[JobActor.Event.ContextBroken] probe.send(frontend, RunJobRequest(s"last", JobParams("path", "MyClass", JsMap.empty, Action.Execute))) probe.expectMsgPF() { @@ -214,7 +205,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -246,7 +237,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -272,7 +263,7 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") name = "name", status = StatusReporter.NOOP, loggersFactory = NOOPLoggerFactory, - connectorStarter = (_, _) => connector, + connectorStarter = (_, _) => Future.successful(connector), jobFactory = ActorF.static(job.ref), defaultInactiveTimeout = 5 minutes ) @@ -318,39 +309,35 @@ class ContextFrontendSpec extends ActorSpec("ctx-frontend-spec") override def release(): Unit = () } - def successfulConnector(): WorkerConnector = { - new WorkerConnector { + def successfulConnector(): Cluster = { + new Cluster { override def whenTerminated(): Future[Unit] = Promise[Unit].future override def askConnection(): Future[PerJobConnection] = Future.successful(NotImplConnection) override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future - override def warmUp(): Unit = () } } - def failedConnection():WorkerConnector = { - new WorkerConnector { + def failedConnection():Cluster = { + new Cluster { override def whenTerminated(): Future[Unit] = Promise[Unit].future override def askConnection(): Future[PerJobConnection] = Future.failed(FilteredException()) override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future - override def warmUp(): Unit = () } } - def crushedConnector(): WorkerConnector = { - new WorkerConnector { + def crushedConnector(): Cluster = { + new Cluster { override def whenTerminated(): Future[Unit] = Promise[Unit].failure(FilteredException()).future override def askConnection(): Future[PerJobConnection] = Promise[PerJobConnection].future - override def warmUp(): Unit = () override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future } } - def oneTimeConnector(future: Future[PerJobConnection]): WorkerConnector = { - new WorkerConnector { + def oneTimeConnector(future: Future[PerJobConnection]): Cluster = { + new Cluster { override def whenTerminated(): Future[Unit] = Promise[Unit].future override def askConnection(): Future[PerJobConnection] = future override def shutdown(force: Boolean): Future[Unit] = Promise[Unit].future - override def warmUp(): Unit = () } } diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala index 04a1e8d8f..216c368aa 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ContextsMasterSpec.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.master.execution import akka.testkit.{TestActorRef, TestProbe} -import io.hydrosphere.mist.core.CommonData.{CancelJobRequest, RunJobRequest} +import io.hydrosphere.mist.common.CommonData.{CancelJobRequest, RunJobRequest} import io.hydrosphere.mist.master.execution.ContextEvent.{CancelJobCommand, RunJobCommand} import io.hydrosphere.mist.master.{ActorSpec, TestData} import io.hydrosphere.mist.utils.akka.ActorF diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala index 59e8e24d2..156a6bb95 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionInfoSpec.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master.execution -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest} +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest} import io.hydrosphere.mist.master.models.JobStartResponse import io.hydrosphere.mist.master.{JobDetails, JobResult, TestUtils} import mist.api.data._ diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala index 6847425d4..4a92b6e0a 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/ExecutionServiceSpec.scala @@ -2,10 +2,9 @@ package io.hydrosphere.mist.master.execution import akka.actor.ActorSystem import akka.testkit.{TestKit, TestProbe} -import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar} +import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar} import io.hydrosphere.mist.master.Messages.StatusMessages.UpdateStatusEvent import io.hydrosphere.mist.master.execution.status.StatusReporter -import io.hydrosphere.mist.master.execution.workers.WorkerHub import io.hydrosphere.mist.master.models.JobStartRequest import io.hydrosphere.mist.master.store.JobRepository import io.hydrosphere.mist.master.{JobDetails, TestData, TestUtils} @@ -29,11 +28,10 @@ class ExecutionServiceSpec extends TestKit(ActorSystem("testMasterService")) it("should start job") { val execution = TestProbe() val repo = mock[JobRepository] - val hub = mock[WorkerHub] val reporter = mock[StatusReporter] when(repo.update(any[JobDetails])).thenSuccess(()) - val service = new ExecutionService(execution.ref, hub, reporter, repo) + val service = new ExecutionService(execution.ref, reporter, repo) val future = service.startJob( JobStartRequest( @@ -62,14 +60,13 @@ class ExecutionServiceSpec extends TestKit(ActorSystem("testMasterService")) //TODO val contextsMaster = TestProbe() val repo = mock[JobRepository] - val hub = mock[WorkerHub] val reporter = mock[StatusReporter] when(repo.get(any[String])) .thenSuccess(Some(mkDetails(JobDetails.Status.Started))) .thenSuccess(Some(mkDetails(JobDetails.Status.Canceled))) - val service = new ExecutionService(contextsMaster.ref, hub, reporter, repo) + val service = new ExecutionService(contextsMaster.ref, reporter, repo) val future = service.stopJob("id") diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala index 3c8f3baa7..ab1489536 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/JobActorSpec.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution import akka.actor.ActorRef import akka.testkit.{TestActorRef, TestProbe} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.Messages.StatusMessages._ import io.hydrosphere.mist.master.execution.status.{ReportedEvent, StatusReporter} import io.hydrosphere.mist.master.execution.workers.{PerJobConnection, WorkerConnection} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala index 7b56a2600..bed922b81 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/SpawnSettingsSpec.scala @@ -1,9 +1,6 @@ package io.hydrosphere.mist.master.execution -import io.hydrosphere.mist.core.CommonData import io.hydrosphere.mist.master.TestData -import io.hydrosphere.mist.master.execution.workers.StopAction -import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter} import io.hydrosphere.mist.master.models.RunMode import org.scalatest.{FunSpec, Matchers} @@ -12,12 +9,7 @@ import scala.concurrent.duration._ class SpawnSettingsSpec extends FunSpec with Matchers with TestData { it("should build worker init info") { - val noop = new WorkerStarter { - override def onStart(name: String, initInfo: CommonData.WorkerInitInfo): WorkerProcess = WorkerProcess.NonLocal - override def stopAction: StopAction = StopAction.Remote - } val spawnSettings = SpawnSettings( - runnerCmd = noop, timeout = 10 seconds, readyTimeout = 10 seconds, akkaAddress = "akkaAddr", @@ -29,15 +21,10 @@ class SpawnSettingsSpec extends FunSpec with Matchers with TestData { val ctx = FooContext.copy(workerMode = RunMode.Shared) val initInfo = spawnSettings.toWorkerInitInfo(ctx) initInfo.sparkConf.toSeq should contain allElementsOf ctx.sparkConf.toSeq - initInfo.maxJobs shouldBe ctx.maxJobs initInfo.downtime shouldBe ctx.downtime initInfo.streamingDuration shouldBe ctx.streamingDuration initInfo.logService shouldBe spawnSettings.logAddress initInfo.masterHttpConf shouldBe spawnSettings.httpAddress initInfo.maxArtifactSize shouldBe spawnSettings.maxArtifactSize - - val excl = ctx.copy(workerMode = RunMode.ExclusiveContext) - val initFoExclusive = spawnSettings.toWorkerInitInfo(excl) - initFoExclusive.maxJobs shouldBe 1 } } diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala index 3bb1c4bb1..f6ccbdb1c 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/status/StatusReporterSpec.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master.execution.status -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.Messages.StatusMessages.{QueuedEvent, UpdateStatusEvent} import io.hydrosphere.mist.master.logging.{JobLogger, LogService} import io.hydrosphere.mist.master.store.JobRepository diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala similarity index 50% rename from mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala rename to mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala index 47206e0c8..546541c3e 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerConnectorSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ClusterSpec.scala @@ -2,24 +2,22 @@ package io.hydrosphere.mist.master.execution.workers import akka.testkit.TestProbe import io.hydrosphere.mist.master.ActorSpec -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.ActorBasedWorkerConnector +import io.hydrosphere.mist.master.execution.Cluster +import io.hydrosphere.mist.master.execution.Cluster.ActorBasedWorkerConnector import scala.concurrent.Promise -class WorkerConnectorSpec extends ActorSpec("actor-based-connector") { +class ClusterSpec extends ActorSpec("actor-based-connector") { it("should proxy call to actor") { val target = TestProbe() val connector = new ActorBasedWorkerConnector(target.ref, Promise[Unit].future) connector.askConnection() - target.expectMsgType[WorkerConnector.Event.AskConnection] - - connector.warmUp() - target.expectMsgType[WorkerConnector.Event.WarmUp.type] + target.expectMsgType[Cluster.Event.AskConnection] connector.shutdown(true) - target.expectMsgType[WorkerConnector.Event.Shutdown] + target.expectMsgType[Cluster.Event.Shutdown] } } diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala index 05945e038..669edf5dd 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/ExclusiveConnectorSpec.scala @@ -2,9 +2,9 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.ActorRef import akka.testkit.{TestActorRef, TestProbe} -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.master.execution.workers.WorkerBridge.Event.CompleteAndShutdown -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released +import io.hydrosphere.mist.master.execution.Cluster import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData} import scala.concurrent.{Await, Future, Promise} @@ -21,7 +21,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData { val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) intercept[Throwable] { Await.result(resolve.future, Duration.Inf) @@ -40,7 +40,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData { val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) val connection = Await.result(resolve.future, Duration.Inf) @@ -68,7 +68,7 @@ class ExclusiveConnectorSpec extends ActorSpec("excl-conn") with TestData { val wrapped = ExclusiveConnector.wrappedConnection(connector.ref, connection) wrapped.release() - connector.expectMsgType[WorkerConnector.Event.Released] + connector.expectMsgType[Cluster.Event.Released] wrapped.run(mkRunReq("id"), ActorRef.noSender) connRef.expectMsgType[RunJobRequest] diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala index e1dc6476f..b8ca22e9e 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/SharedConnectorSpec.scala @@ -4,8 +4,9 @@ import java.util.concurrent.atomic.AtomicInteger import akka.actor.ActorRef import akka.testkit.{TestActorRef, TestProbe} -import io.hydrosphere.mist.core.CommonData.RunJobRequest -import io.hydrosphere.mist.master.execution.workers.WorkerConnector.Event.Released +import io.hydrosphere.mist.common.CommonData.RunJobRequest +import io.hydrosphere.mist.master.execution.Cluster +import io.hydrosphere.mist.master.models.RunMode import io.hydrosphere.mist.master.{ActorSpec, TestData} import org.scalatest.Matchers import org.scalatest.concurrent.Eventually @@ -33,14 +34,14 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) val connection1 = Await.result(resolve.future, Duration.Inf) val resolve2 = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve2)) + probe.send(connector, Cluster.Event.AskConnection(resolve2)) val connection2 = Await.result(resolve2.future, Duration.Inf) val resolve3 = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve3)) + probe.send(connector, Cluster.Event.AskConnection(resolve3)) connection1.id shouldBe "1" connection2.id shouldBe "2" @@ -57,7 +58,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val remote = TestProbe() val connector = TestActorRef[SharedConnector](SharedConnector.props( id = "id", - ctx = FooContext, + ctx = FooContext.copy(workerMode = RunMode.Shared, precreated = true), startConnection = (id, ctx) => { val x = callCounter.incrementAndGet() val conn = WorkerConnection(x.toString, remote.ref, workerLinkData, Promise[Unit].future) @@ -66,9 +67,8 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te )) val probe = TestProbe() - probe.send(connector, WorkerConnector.Event.WarmUp) callCounter.get() shouldBe 2 - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status = probe.expectMsgType[SharedConnector.ProcessStatus] status.poolSize shouldBe 2 status.requestsSize shouldBe 0 @@ -93,11 +93,11 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) val connection1 = Await.result(resolve.future, Duration.Inf) val resolve2 = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve2)) + probe.send(connector, Cluster.Event.AskConnection(resolve2)) val connection2 = Await.result(resolve2.future, Duration.Inf) connection1.id shouldBe "1" @@ -106,7 +106,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te connection1.release() val resolve3 = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve3)) + probe.send(connector, Cluster.Event.AskConnection(resolve3)) val conn3 = Await.result(resolve3.future, Duration.Inf) conn3.id shouldBe "1" @@ -123,16 +123,16 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) val connection1 = Await.result(resolve.future, Duration.Inf) - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status = probe.expectMsgType[SharedConnector.ProcessStatus] status.inUseSize shouldBe 1 status.poolSize shouldBe 0 termination.success(()) //ConnTerminated is fired - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status2 = probe.expectMsgType[SharedConnector.ProcessStatus] status2.inUseSize shouldBe 0 status2.poolSize shouldBe 0 @@ -149,17 +149,17 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val probe = TestProbe() val resolve = Promise[PerJobConnection] - probe.send(connector, WorkerConnector.Event.AskConnection(resolve)) + probe.send(connector, Cluster.Event.AskConnection(resolve)) val connection1 = Await.result(resolve.future, Duration.Inf) - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status = probe.expectMsgType[SharedConnector.ProcessStatus] status.inUseSize shouldBe 1 status.poolSize shouldBe 0 connection1.release() - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status1 = probe.expectMsgType[SharedConnector.ProcessStatus] status1.poolSize shouldBe 1 status1.inUseSize shouldBe 0 @@ -167,7 +167,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te termination.success(()) //ConnTerminated is fired - probe.send(connector, WorkerConnector.Event.GetStatus) + probe.send(connector, Cluster.Event.GetStatus) val status2 = probe.expectMsgType[SharedConnector.ProcessStatus] status2.poolSize shouldBe 0 status2.inUseSize shouldBe 0 @@ -187,7 +187,7 @@ class SharedConnectorSpec extends ActorSpec("shared-conn") with Matchers with Te val connector = TestProbe() val wrapped = SharedConnector.wrappedConnection(connector.ref, connection) wrapped.release() - connector.expectMsgType[Released] + connector.expectMsgType[Cluster.Event.Released] connRef.expectNoMessage(1 second) wrapped.run(mkRunReq("id"), ActorRef.noSender) diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala index 4a21c123a..65c11c957 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerBridgeSpec.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.master.execution.workers import java.util.concurrent.atomic.AtomicBoolean import akka.testkit.TestProbe -import io.hydrosphere.mist.core.CommonData._ +import io.hydrosphere.mist.common.CommonData._ import io.hydrosphere.mist.master.execution.workers.WorkerBridge.Event.CompleteAndShutdown import io.hydrosphere.mist.master.{ActorSpec, TestData, TestUtils} import mist.api.data._ diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala index 4769a88ae..a8c474790 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerHubSpec.scala @@ -1,8 +1,9 @@ package io.hydrosphere.mist.master.execution.workers import akka.actor.ActorRef -import io.hydrosphere.mist.core.{CommonData, MockitoSugar} +import io.hydrosphere.mist.common.{CommonData, MockitoSugar} import io.hydrosphere.mist.master.TestData +import io.hydrosphere.mist.master.execution.Cluster import io.hydrosphere.mist.master.models.ContextConfig import org.scalatest.concurrent.Eventually import org.scalatest.time.{Seconds, Span} @@ -11,46 +12,46 @@ import org.scalatest.{FunSpec, Matchers} import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future, Promise} -class WorkerHubSpec extends FunSpec with Matchers with TestData with Eventually with MockitoSugar { - - it("should mirror connections") { - val termination = Promise[Unit] - - val runner = new WorkerRunner { - override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = - Future.successful(WorkerConnection(id, null, workerLinkData.copy(name = id), termination.future)) - } - val hub = new WorkerHub(runner, TestConnector.apply) - - val connector = hub.start("id", FooContext) - - Await.result(connector.askConnection(), Duration.Inf) - - eventually(timeout(Span(3, Seconds))) { - hub.workerConnections().size shouldBe 1 - } - termination.success(()) - eventually(timeout(Span(3, Seconds))) { - hub.workerConnections().size shouldBe 0 - } - } - - - case class TestConnector( - id: String, - ctx: ContextConfig, - runner: WorkerRunner) extends WorkerConnector { - - import scala.concurrent.ExecutionContext.Implicits.global - - def askConnection(): Future[PerJobConnection] = runner(id, ctx).map(conn => mock[PerJobConnection]) - - def warmUp(): Unit = () - - def shutdown(force: Boolean): Future[Unit] = ??? - - def whenTerminated(): Future[Unit] = ??? - - def releaseConnection(connectionId: String): Unit = ??? - } -} +//class WorkerHubSpec extends FunSpec with Matchers with TestData with Eventually with MockitoSugar { +// +// it("should mirror connections") { +// val termination = Promise[Unit] +// +// val runner = new WorkerRunner { +// override def apply(id: String, ctx: ContextConfig): Future[WorkerConnection] = +// Future.successful(WorkerConnection(id, null, workerLinkData.copy(name = id), termination.future)) +// } +// val hub = new WorkerHub(runner, TestConnector.apply) +// +// val connector = hub.start("id", FooContext) +// +// Await.result(connector.askConnection(), Duration.Inf) +// +// eventually(timeout(Span(3, Seconds))) { +// hub.workerConnections().size shouldBe 1 +// } +// termination.success(()) +// eventually(timeout(Span(3, Seconds))) { +// hub.workerConnections().size shouldBe 0 +// } +// } +// +// +// case class TestConnector( +// id: String, +// ctx: ContextConfig, +// runner: WorkerRunner) extends Cluster { +// +// import scala.concurrent.ExecutionContext.Implicits.global +// +// def askConnection(): Future[PerJobConnection] = runner(id, ctx).map(conn => mock[PerJobConnection]) +// +// def warmUp(): Unit = () +// +// def shutdown(force: Boolean): Future[Unit] = ??? +// +// def whenTerminated(): Future[Unit] = ??? +// +// def releaseConnection(connectionId: String): Unit = ??? +// } +//} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala index b59967caf..1698aa354 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/WorkerRunnerSpec.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.master.execution.workers import java.util.concurrent.atomic.AtomicBoolean -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.master.execution.workers.starter.{WorkerProcess, WorkerStarter} import io.hydrosphere.mist.master.execution.{SpawnSettings, workers} import io.hydrosphere.mist.master.{ActorSpec, FilteredException, TestData} @@ -20,7 +20,6 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc describe("default runner") { def mkSpawnSettings(starter: WorkerStarter): SpawnSettings = SpawnSettings( - runnerCmd = starter, timeout = 10 seconds, readyTimeout = 10 seconds, akkaAddress = "akkaAddr", @@ -41,7 +40,8 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc val runner = new workers.WorkerRunner.DefaultRunner( spawn = mkSpawnSettings(starter), regHub = regHub, - connect = (_, _, _, _, _) => Future.successful(WorkerConnection("id", null, workerLinkData, termination.future)) + connect = (_, _, _, _, _) => Future.successful(WorkerConnection("id", null, workerLinkData, termination.future)), + runnerCmd = starter ) Await.result(runner("id", FooContext), Duration.Inf) @@ -61,7 +61,8 @@ class WorkerRunnerSpec extends ActorSpec("worker-runner") with TestData with Moc val runner = new workers.WorkerRunner.DefaultRunner( spawn = mkSpawnSettings(runnerCmd), regHub = regHub, - connect = (_, _, _, _, _) => Future.failed(FilteredException()) + connect = (_, _, _, _, _) => Future.failed(FilteredException()), + runnerCmd = runnerCmd ) intercept[Throwable] { diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala index a2c56623d..d8660cbe9 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/execution/workers/starter/SparkSubmitBuilderSpec.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.master.execution.workers.starter -import io.hydrosphere.mist.core.CommonData.WorkerInitInfo +import io.hydrosphere.mist.common.CommonData.WorkerInitInfo import io.hydrosphere.mist.master.TestData import org.scalatest.{FunSpec, Matchers} import scala.concurrent.duration._ @@ -11,7 +11,6 @@ class SparkSubmitBuilderSpec extends FunSpec with Matchers { val testInfo = WorkerInitInfo( sparkConf = Map("spark.master" -> "spark://localhost:4433", "a.b.c" -> "xyz"), - maxJobs = 10, downtime = 1 second, streamingDuration = 1 second, logService = "localhost:2005", diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala index 7f395701b..ee4b10846 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/DevApiSpec.scala @@ -4,9 +4,9 @@ import mist.api.data._ import mist.api.encoding.JsSyntax._ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.testkit.ScalatestRouteTest -import io.hydrosphere.mist.core.CommonData.{JobParams, RunJobRequest} -import io.hydrosphere.mist.core.MockitoSugar -import io.hydrosphere.mist.core.CommonData.Action +import io.hydrosphere.mist.common.CommonData.{JobParams, RunJobRequest} +import io.hydrosphere.mist.common.MockitoSugar +import io.hydrosphere.mist.common.CommonData.Action import io.hydrosphere.mist.master.JobDetails.{Source, Status} import io.hydrosphere.mist.master.execution.ExecutionInfo import io.hydrosphere.mist.master.MainService diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala index 10b29198e..5f646e5fd 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/HttpApiV2Spec.scala @@ -7,12 +7,12 @@ import java.util.UUID import akka.http.scaladsl.model._ import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.util.ByteString -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.{FunctionInfoData, MockitoSugar} +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.{FunctionInfoData, MockitoSugar} import io.hydrosphere.mist.master.JobDetails.Source import io.hydrosphere.mist.master._ import io.hydrosphere.mist.master.artifact.ArtifactRepository -import io.hydrosphere.mist.master.execution.{ExecutionService, WorkerLink} +import io.hydrosphere.mist.master.execution.ExecutionService import io.hydrosphere.mist.master.data.{ContextsStorage, FunctionConfigStorage} import io.hydrosphere.mist.master.interfaces.JsonCodecs import io.hydrosphere.mist.master.jobs.FunctionsService @@ -48,75 +48,75 @@ class HttpApiV2Spec extends FunSpec } } - describe("workers") { - - it("should return workers") { - val execution = mock[ExecutionService] - when(execution.workers()).thenReturn(Seq(workerLinkData)) - - val route = HttpV2Routes.workerRoutes(execution) - - Get("/v2/api/workers") ~> route ~> check { - status shouldBe StatusCodes.OK - val rsp = responseAs[Seq[WorkerLink]] - rsp.size shouldBe 1 - } - } - - it("should stop worker") { - val execution = mock[ExecutionService] - when(execution.stopWorker(any[String])).thenSuccess(()) - - val route = HttpV2Routes.workerRoutes(execution) - - Delete("/v2/api/workers/id") ~> route ~> check { - status shouldBe StatusCodes.OK - } - } - - it("should get full worker info") { - val execution = mock[ExecutionService] - when(execution.getWorkerLink(any[String])) - .thenReturn(Some(WorkerLink( - "id", "test", None, - WorkerInitInfo(Map(), 20, Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, "")))) - - val route = HttpV2Routes.workerRoutes(execution) - - Get("/v2/api/workers/id") ~> route ~> check { - status shouldBe StatusCodes.OK - val resp = responseAs[WorkerLink] - resp.name shouldBe "id" - resp.initInfo shouldBe WorkerInitInfo(Map(), 20, Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "") - resp.sparkUi should not be defined - resp.address shouldBe "test" - } - } - - it("should return worker jobs") { - val execution = mock[ExecutionService] - when(execution.getHistory(any[JobDetailsRequest])).thenSuccess(JobDetailsResponse( - Seq(JobDetails("id", "1", - JobParams("path", "className", JsMap.empty, Action.Execute), - "context", None, JobDetails.Source.Http) - ), 1 - )) - - val route = HttpV2Routes.workerRoutes(execution) - - Get("/v2/api/workers/id/jobs?status=started") ~> route ~> check { - status shouldBe StatusCodes.OK - val jobs = responseAs[Seq[JobDetails]] - jobs.size shouldBe 1 - } - Get("/v2/api/workers/id/jobs?status=started&paginate=true") ~> route ~> check { - status shouldBe StatusCodes.OK - val rsp = responseAs[JobDetailsResponse] - rsp.jobs.size shouldBe 1 - } - } - - } +// describe("workers") { +// +// it("should return workers") { +// val execution = mock[ExecutionService] +// when(execution.workers()).thenReturn(Seq(workerLinkData)) +// +// val route = HttpV2Routes.workerRoutes(execution) +// +// Get("/v2/api/workers") ~> route ~> check { +// status shouldBe StatusCodes.OK +// val rsp = responseAs[Seq[WorkerLink]] +// rsp.size shouldBe 1 +// } +// } +// +// it("should stop worker") { +// val execution = mock[ExecutionService] +// when(execution.stopWorker(any[String])).thenSuccess(()) +// +// val route = HttpV2Routes.workerRoutes(execution) +// +// Delete("/v2/api/workers/id") ~> route ~> check { +// status shouldBe StatusCodes.OK +// } +// } +// +// it("should get full worker info") { +// val execution = mock[ExecutionService] +// when(execution.getWorkerLink(any[String])) +// .thenReturn(Some(WorkerLink( +// "id", "test", None, +// WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0", "localhost:0", 262144000, "")))) +// +// val route = HttpV2Routes.workerRoutes(execution) +// +// Get("/v2/api/workers/id") ~> route ~> check { +// status shouldBe StatusCodes.OK +// val resp = responseAs[WorkerLink] +// resp.name shouldBe "id" +// resp.initInfo shouldBe WorkerInitInfo(Map(), Duration.Inf, Duration.Inf, "test", "localhost:0","localhost:0", 262144000, "") +// resp.sparkUi should not be defined +// resp.address shouldBe "test" +// } +// } +// +// it("should return worker jobs") { +// val execution = mock[ExecutionService] +// when(execution.getHistory(any[JobDetailsRequest])).thenSuccess(JobDetailsResponse( +// Seq(JobDetails("id", "1", +// JobParams("path", "className", JsMap.empty, Action.Execute), +// "context", None, JobDetails.Source.Http) +// ), 1 +// )) +// +// val route = HttpV2Routes.workerRoutes(execution) +// +// Get("/v2/api/workers/id/jobs?status=started") ~> route ~> check { +// status shouldBe StatusCodes.OK +// val jobs = responseAs[Seq[JobDetails]] +// jobs.size shouldBe 1 +// } +// Get("/v2/api/workers/id/jobs?status=started&paginate=true") ~> route ~> check { +// status shouldBe StatusCodes.OK +// val rsp = responseAs[JobDetailsResponse] +// rsp.jobs.size shouldBe 1 +// } +// } +// +// } describe("functions") { diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala index 428928969..ca21f8f2a 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/interfaces/http/WsApiSpec.scala @@ -4,7 +4,7 @@ import mist.api.data._ import akka.http.scaladsl.testkit.{ScalatestRouteTest, WSProbe} import akka.stream.ActorMaterializer import akka.stream.scaladsl.Source -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.Messages.StatusMessages._ import io.hydrosphere.mist.master.EventsStreamer import org.mockito.Mockito._ diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala index 113ef5155..96a8ce4a2 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/jobs/FunctionsServiceSpec.scala @@ -5,9 +5,9 @@ import java.nio.file.Paths import akka.actor.{ActorSystem, Status} import akka.testkit.{TestKit, TestProbe} -import io.hydrosphere.mist.core.CommonData.{Action, GetAllFunctions, GetFunctionInfo, ValidateFunctionParameters} -import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData, MockitoSugar} -import io.hydrosphere.mist.core.ExtractedFunctionData +import io.hydrosphere.mist.common.CommonData.{Action, GetAllFunctions, GetFunctionInfo, ValidateFunctionParameters} +import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData, MockitoSugar} +import io.hydrosphere.mist.common.ExtractedFunctionData import io.hydrosphere.mist.master.TestData import io.hydrosphere.mist.master.artifact.ArtifactRepository import io.hydrosphere.mist.master.data.{Contexts, ContextsStorage, FunctionConfigStorage} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala index 697f641df..579bdfc71 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogStreamsSpec.scala @@ -4,8 +4,8 @@ import akka.actor.ActorSystem import akka.stream.ActorMaterializer import akka.stream.scaladsl.{Keep, Sink, Source} import akka.testkit.TestKit -import io.hydrosphere.mist.core.MockitoSugar -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.MockitoSugar +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.FilteredException import org.mockito.Mockito.verify import org.scalatest.{FunSpecLike, Matchers} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala index 32150bbb8..8aa3d4203 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/logging/LogWriterSpec.scala @@ -7,7 +7,7 @@ import akka.pattern.ask import akka.testkit.{TestActorRef, TestKit} import akka.util.Timeout import com.typesafe.config.ConfigFactory -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.master.LogStoragePaths import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfterAll, FunSpecLike, Matchers} diff --git a/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala b/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala index 7c83542ef..8f000249f 100644 --- a/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala +++ b/mist/master/src/test/scala/io/hydrosphere/mist/master/store/H2RepoSpec.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.master.store import java.nio.file.Paths -import io.hydrosphere.mist.core.CommonData.{Action, JobParams} +import io.hydrosphere.mist.common.CommonData.{Action, JobParams} import io.hydrosphere.mist.master.JobDetails import io.hydrosphere.mist.master.JobDetails.Source import mist.api.data.JsMap diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala index 6bfc0a1a8..2d9d17812 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoExtractor.scala @@ -3,10 +3,10 @@ package io.hydrosphere.mist.job import java.io.File import java.net.URLClassLoader -import io.hydrosphere.mist.core -import io.hydrosphere.mist.core.CommonData.{Action, EnvInfo} -import io.hydrosphere.mist.core.{ExtractedFunctionData, FunctionInfoData} -import io.hydrosphere.mist.core.ExtractedFunctionData +import io.hydrosphere.mist.common +import io.hydrosphere.mist.common.CommonData.{Action, EnvInfo} +import io.hydrosphere.mist.common.{ExtractedFunctionData, FunctionInfoData} +import io.hydrosphere.mist.common.ExtractedFunctionData import io.hydrosphere.mist.python.{FunctionInfoPythonExecutor, PythonCmd} import io.hydrosphere.mist.utils.{Err, Logger, Succ, TryLoad} import io.hydrosphere.mist.utils.{Err, Succ, TryLoad} @@ -36,7 +36,7 @@ class JvmFunctionInfoExtractor(mkLoader: ClassLoader => FunctionInstanceLoader) val executeFnInstance = loader.loadFnInstance(className, Action.Execute) executeFnInstance orElse loader.loadFnInstance(className, Action.Serve) map { instance => - FunctionInfo(instance, core.ExtractedFunctionData( + FunctionInfo(instance, common.ExtractedFunctionData( lang = instance.lang, execute = instance.describe().collect { case x: UserInputArgument => x }, isServe = !executeFnInstance.isSuccess, diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala index ef1c201fe..c2e04db3e 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProvider.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.job import akka.actor.{ActorRef, ActorSystem, PoisonPill} import com.typesafe.config.ConfigFactory -import io.hydrosphere.mist.core.CommonData -import io.hydrosphere.mist.core.CommonData.RegisterJobInfoProvider +import io.hydrosphere.mist.common.CommonData +import io.hydrosphere.mist.common.CommonData.RegisterJobInfoProvider import io.hydrosphere.mist.utils.Logger import io.hydrosphere.mist.utils.akka.WhenTerminated diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala index fe988128a..61617adc0 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInfoProviderActor.scala @@ -4,8 +4,8 @@ import java.io.File import akka.actor.SupervisorStrategy.Resume import akka.actor.{Status, _} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.ExtractedFunctionData +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.ExtractedFunctionData import io.hydrosphere.mist.utils.{Err, Succ, TryLoad} import mist.api.{Extracted, Failed} import mist.api.data.JsMap diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala index a333bdb1d..1dab4276a 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/FunctionInstanceLoader.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.job import java.io.File import java.net.URLClassLoader -import io.hydrosphere.mist.core.CommonData.Action +import io.hydrosphere.mist.common.CommonData.Action import io.hydrosphere.mist.job import io.hydrosphere.mist.utils.{Err, TryLoad} diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala index 775a5fcb3..77ef8470a 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/job/JvmLangDetector.scala @@ -2,7 +2,7 @@ package io.hydrosphere.mist.job import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm._ import com.esotericsoftware.reflectasm.shaded.org.objectweb.asm.Opcodes._ -import io.hydrosphere.mist.core.FunctionInfoData +import io.hydrosphere.mist.common.FunctionInfoData class CheckSource extends ClassVisitor(ASM5) { diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala index 2c10668c8..663d4a62f 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/python/PythonExecuter.scala @@ -3,8 +3,8 @@ package io.hydrosphere.mist.python import java.io.File import java.nio.file.Paths -import io.hydrosphere.mist.core.CommonData.{EnvInfo, RunJobRequest} -import io.hydrosphere.mist.core.PythonEntrySettings +import io.hydrosphere.mist.common.CommonData.{EnvInfo, RunJobRequest} +import io.hydrosphere.mist.common.PythonEntrySettings import io.hydrosphere.mist.utils.Logger import io.hydrosphere.mist.worker.MistScContext import io.hydrosphere.mist.worker.runners.python.wrappers.{ConfigurationWrapper, SparkStreamingWrapper} diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala index e7caa7163..b84632e13 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/MasterBridge.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.worker import java.nio.file.Path import akka.actor._ -import io.hydrosphere.mist.core.CommonData._ +import io.hydrosphere.mist.common.CommonData._ import io.hydrosphere.mist.utils.akka.{ActorF, ActorFSyntax, ActorRegHub} import io.hydrosphere.mist.worker.MasterBridge.{AppShutdown, ReceiveInitTimeout} import io.hydrosphere.mist.worker.logging.RemoteLogsWriter diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala index 373bc6e4e..9cadb40c2 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/RequestSetup.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.worker -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.worker.logging.{LogsWriter, RemoteAppender, RemoteLogsWriter} import org.apache.log4j.{LogManager, Logger} diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala index d424ec2af..f6bdd485e 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/WorkerActor.scala @@ -4,7 +4,7 @@ import java.io.{PrintWriter, StringWriter} import akka.actor._ import akka.pattern.pipe -import io.hydrosphere.mist.core.CommonData._ +import io.hydrosphere.mist.common.CommonData._ import io.hydrosphere.mist.worker.runners._ import mist.api.data.JsData import org.apache.spark.streaming.StreamingContext diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala index 294c2a072..702e986b8 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteAppender.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.worker.logging -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.logging.LogEvent import org.apache.log4j.spi.LoggingEvent import org.apache.log4j.{AppenderSkeleton, Level, SimpleLayout} diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala index aa29270f2..cf6e967b2 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/logging/RemoteLogsWriter.scala @@ -8,7 +8,7 @@ import akka.stream.scaladsl.{Keep, Sink, Source, Tcp} import akka.util.ByteString import com.twitter.chill.{KryoPool, ScalaKryoInstantiator} import com.typesafe.config.ConfigFactory -import io.hydrosphere.mist.core.logging.{Level, LogEvent} +import io.hydrosphere.mist.common.logging.{Level, LogEvent} import io.hydrosphere.mist.worker.logging.RemoteLogsWriter.Key import org.slf4j.LoggerFactory diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala index 0fc2673d5..342625103 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/JobRunner.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.worker.runners -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.worker.MistScContext import mist.api.data.JsData diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala index af29d6001..cb4f3f173 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/ScalaRunner.scala @@ -3,7 +3,7 @@ package io.hydrosphere.mist.worker.runners import java.io.File import java.net.URL -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.job.FunctionInstanceLoader import io.hydrosphere.mist.utils.EitherOps._ import io.hydrosphere.mist.utils.{Err, Succ} diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala index 13d916db4..05174a022 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/PythonRunner.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.worker.runners.python -import io.hydrosphere.mist.core.CommonData.RunJobRequest +import io.hydrosphere.mist.common.CommonData.RunJobRequest import io.hydrosphere.mist.python.PythonFunctionExecutor import io.hydrosphere.mist.utils.Logger import io.hydrosphere.mist.worker.runners.JobRunner diff --git a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala index 7e0d90e4f..5fe874b11 100644 --- a/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala +++ b/mist/worker/src/main/scala/io/hydrosphere/mist/worker/runners/python/wrappers/ConfigurationWrapper.scala @@ -1,6 +1,6 @@ package io.hydrosphere.mist.worker.runners.python.wrappers -import io.hydrosphere.mist.core.CommonData.JobParams +import io.hydrosphere.mist.common.CommonData.JobParams import io.hydrosphere.mist.utils.Collections import mist.api.data.JsData diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala index 1cc2eaaf4..a115b9d23 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoExtractorSpec.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.job import java.io.File -import io.hydrosphere.mist.core.CommonData.{Action, EnvInfo} -import io.hydrosphere.mist.core.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings} +import io.hydrosphere.mist.common.CommonData.{Action, EnvInfo} +import io.hydrosphere.mist.common.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings} import io.hydrosphere.mist.utils.{Err, Succ} import mist.api.{InternalArgument, MInt, UserInputArgument} import org.mockito.Matchers.{endsWith => mockitoEndsWith, eq => mockitoEq} diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala index 094fb9b75..62872fd4e 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/job/FunctionInfoProviderActorSpec.scala @@ -5,8 +5,8 @@ import java.nio.file.Paths import akka.actor.{Actor, ActorSystem, Status} import akka.testkit.{TestActorRef, TestKit, TestProbe} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings} +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.{ExtractedFunctionData, MockitoSugar, PythonEntrySettings} import io.hydrosphere.mist.utils.{Err, Succ} import mist.api._ import mist.api.data._ diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala index 08c1517bf..bb09d3f5e 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/MasterBridgeSpec.scala @@ -2,8 +2,8 @@ package io.hydrosphere.mist.worker import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit, TestProbe} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.utils.akka.{ActorF, ActorRegHub} import mist.api.data.JsMap import org.apache.spark.SparkConf @@ -18,7 +18,7 @@ class MasterBridgeSpec extends TestKit(ActorSystem("WorkerBridgeSpec")) with BeforeAndAfterAll { def mkInitInfo(sparkConf: Map[String, String]) = - WorkerInitInfo(sparkConf, 1, 20 seconds, 20 seconds, "localhost:2005", "localhost:2003", "localhost:2004", 202020, "") + WorkerInitInfo(sparkConf, 20 seconds, 20 seconds, "localhost:2005", "localhost:2003", "localhost:2004", 202020, "") it("should create named context with spark.streaming.stopSparkContextByDefault=false") { val sparkConf = Map( diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala index 8b721dad0..534109f60 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/RequestSetupSpec.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.worker -import io.hydrosphere.mist.core.CommonData.{Action, JobParams, RunJobRequest} -import io.hydrosphere.mist.core.logging.LogEvent +import io.hydrosphere.mist.common.CommonData.{Action, JobParams, RunJobRequest} +import io.hydrosphere.mist.common.logging.LogEvent import io.hydrosphere.mist.worker.logging.LogsWriter import mist.api.data.JsMap import org.apache.log4j.LogManager diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala index e0b9386fd..7a5c9d1d3 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/WorkerActorSpec.scala @@ -4,8 +4,8 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit, TestProbe} -import io.hydrosphere.mist.core.CommonData._ -import io.hydrosphere.mist.core.MockitoSugar +import io.hydrosphere.mist.common.CommonData._ +import io.hydrosphere.mist.common.MockitoSugar import io.hydrosphere.mist.worker.runners.{ArtifactDownloader, JobRunner, RunnerSelector} import mist.api.data.{JsData, _} import mist.api.encoding.defaultEncoders._ diff --git a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala index ab44a61aa..e4a13f1e9 100644 --- a/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala +++ b/mist/worker/src/test/scala/io/hydrosphere/mist/worker/logging/RemoteAppenderSpec.scala @@ -1,7 +1,7 @@ package io.hydrosphere.mist.worker.logging -import io.hydrosphere.mist.core.MockitoSugar -import io.hydrosphere.mist.core.logging._ +import io.hydrosphere.mist.common.MockitoSugar +import io.hydrosphere.mist.common.logging._ import org.apache.log4j.spi.{LoggingEvent, NOPLogger, NOPLoggerRepository} import org.apache.log4j.{Category, Level => L4jLevel} import org.mockito.Matchers.{eq => mockitoEq} diff --git a/project/Library.scala b/project/Library.scala index a5b8dc761..5f92dc556 100644 --- a/project/Library.scala +++ b/project/Library.scala @@ -4,6 +4,8 @@ object Library { val slf4j = "org.slf4j" % "slf4j-api" % "1.7.5" val slf4jLog4j = "org.slf4j" % "slf4j-log4j12" % "1.7.5" + val log4j = "log4j" % "log4j" % "1.2.17" + val log4jExtras = "log4j" % "apache-log4j-extras" % "1.2.17" val scopt = "com.github.scopt" %% "scopt" % "3.6.0" val typesafeConfig = "com.typesafe" % "config" % "1.3.1" @@ -16,7 +18,8 @@ object Library { val pahoMqtt = "org.eclipse.paho" % "org.eclipse.paho.client.mqttv3" % "1.1.0" val kafka = "org.apache.kafka" %% "kafka" % "0.10.2.0" exclude("log4j", "log4j") exclude("org.slf4j","slf4j-log4j12") - val cats = "org.typelevel" %% "cats" % "0.9.0" + val cats = "org.typelevel" %% "cats-core" % "1.3.1" + val catsEffect = "org.typelevel" %% "cats-effect" % "1.0.0" val scalaTest = "org.scalatest" %% "scalatest" % "3.0.1" val junit = "junit" % "junit" % "4.12" @@ -53,9 +56,13 @@ object Library { "org.apache.spark" %% "spark-core" % v, "org.apache.spark" %% "spark-sql" % v, "org.apache.spark" %% "spark-hive" % v, - "org.apache.spark" %% "spark-streaming" % v, + "org.apache.spark" %% "spark-streaming" % v ) val jsr305 = "com.google.code.findbugs" % "jsr305" % "1.3.9" + val scalaSsh = "com.decodified" %% "scala-ssh" % "0.9.0" + val awsSdkEC2 = "software.amazon.awssdk" % "ec2" % "2.0.0-preview-10" + val awsSdkEMR = "software.amazon.awssdk" % "emr" % "2.0.0-preview-10" + val awsSdkIAM = "software.amazon.awssdk" % "iam" % "2.0.0-preview-10" } diff --git a/project/plugins.sbt b/project/plugins.sbt index c73544570..e67014937 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,4 @@ +addSbtPlugin("org.lyranthe.sbt" % "partial-unification" % "1.1.2") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.2")