diff --git a/pom.xml b/pom.xml index 8a1f54d1..aff1e656 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,11 @@ plain-credentials 1.7 + + org.apache.commons + commons-collections4 + 4.4 + diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index f1a6fa41..f64ab14c 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -7,6 +7,10 @@ import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.model.azure.*; import hudson.plugins.spotinst.model.gcp.*; +import hudson.plugins.spotinst.model.redis.UnlockGroupControllerResponse; +import hudson.plugins.spotinst.model.redis.GetGroupControllerLockResponse; +import hudson.plugins.spotinst.model.redis.LockGroupControllerRequest; +import hudson.plugins.spotinst.model.redis.LockGroupControllerResponse; import jenkins.model.Jenkins; import org.apache.commons.httpclient.HttpStatus; import org.slf4j.Logger; @@ -19,7 +23,7 @@ public class SpotinstApi { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstApi.class); - private final static String SPOTINST_API_HOST = "https://api.spotinst.io"; + private final static String SPOTINST_API_HOST = "http://localhost:3100"; private final static String HEADER_AUTH = "Authorization"; private final static String AUTH_PREFIX = "Bearer "; private final static String HEADER_CONTENT_TYPE = "Content-Type"; @@ -89,8 +93,9 @@ public static AwsScaleUpResult awsScaleUp(String groupId, int adjustment, String Map queryParams = buildQueryParams(accountId); queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers, + queryParams); AwsScaleUpResponse scaleUpResponse = getCastedResponse(response, AwsScaleUpResponse.class); @@ -125,13 +130,12 @@ public static List getAllAwsInstanceTypes(String accountId) thr List retVal; Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - queryParams.put("distinctTypesList","true"); + queryParams.put("distinctTypesList", "true"); - RestResponse response = - RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams); + RestResponse response = RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams); - AwsInstanceTypesResponse - allAwsInstanceTypesResponse = getCastedResponse(response, AwsInstanceTypesResponse.class); + AwsInstanceTypesResponse allAwsInstanceTypesResponse = + getCastedResponse(response, AwsInstanceTypesResponse.class); retVal = allAwsInstanceTypesResponse.getResponse().getItems(); @@ -149,8 +153,9 @@ public static GcpScaleUpResult gcpScaleUp(String groupId, int adjustment, String Map queryParams = buildQueryParams(accountId); queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers, + queryParams); GcpScaleUpResponse scaleUpResponse = getCastedResponse(response, GcpScaleUpResponse.class); @@ -171,9 +176,9 @@ public static Boolean gcpDetachInstance(String groupId, String instanceName, Str request.setShouldTerminateInstances(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers, + queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -206,8 +211,9 @@ public static List getAzureGroupInstances(String groupId, Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - RestResponse response = RestClient - .sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers, queryParams); + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers, + queryParams); AzureGroupInstancesResponse instancesResponse = getCastedResponse(response, AzureGroupInstancesResponse.class); @@ -224,9 +230,9 @@ public static Boolean azureScaleUp(String groupId, int adjustment, String accoun Map queryParams = buildQueryParams(accountId); queryParams.put("adjustment", String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers, + queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -243,9 +249,9 @@ public static Boolean azureDetachInstance(String groupId, String instanceId, Str request.setShouldDecrementTargetCapacity(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body, + headers, queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -260,9 +266,9 @@ public static AzureGroupStatus getAzureVmGroupStatus(String groupId, String acco Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - RestResponse response = RestClient - .sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status", headers, - queryParams); + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status", + headers, queryParams); AzureGroupStatusResponse vmsResponse = getCastedResponse(response, AzureGroupStatusResponse.class); @@ -281,9 +287,9 @@ public static List azureVmScaleUp(String groupId, int a Map queryParams = buildQueryParams(accountId); queryParams.put("adjustment", String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up", null, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up", + null, headers, queryParams); AzureScaleUpResponse scaleUpResponse = getCastedResponse(response, AzureScaleUpResponse.class); @@ -303,9 +309,9 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI request.setShouldDecrementTargetCapacity(true); request.setShouldTerminateVms(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms", body, - headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms", + body, headers, queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -313,6 +319,67 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI } //endregion + //Redis + public static String getGroupLockValueById(String groupId, String accountId) throws ApiException { + String retVal = null; + + Map headers = buildHeaders(); + + Map queryParams = buildQueryParams(accountId); + + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers, + queryParams); + + GetGroupControllerLockResponse lockResponse = getCastedResponse(response, GetGroupControllerLockResponse.class); + + if (CollectionUtils.isEmpty(lockResponse.getResponse().getItems()) == false) { + retVal = lockResponse.getResponse().getItems().get(0); + } + + return retVal; + } + + public static String LockGroupController(String lockKey, String accountId, String lockValue, + Integer ttl) throws ApiException { + String retVal = null; + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); + LockGroupControllerRequest request = new LockGroupControllerRequest(lockKey, lockValue, ttl); + String body = JsonMapper.toJson(request); + + RestResponse response = + RestClient.sendPost(SPOTINST_API_HOST + "/aws/ec2/group/jenkinsPlugin/lock", body, headers, + queryParams); + + LockGroupControllerResponse lockControllerValue = + getCastedResponse(response, LockGroupControllerResponse.class); + //TODO: check optimizer and service response + if (lockControllerValue.getResponse().getItems().size() > 0) { + retVal = lockControllerValue.getResponse().getItems().get(0); + } + + return retVal; + } + + public static Integer UnlockGroupController(String groupId, String accountId) throws ApiException { + Integer retVal = null; + + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); + RestResponse response = + RestClient.sendDelete(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers, + queryParams); + UnlockGroupControllerResponse redisValue = getCastedResponse(response, UnlockGroupControllerResponse.class); + + if (redisValue.getResponse().getItems().size() > 0) { + retVal = redisValue.getResponse().getItems().get(0); + } + + return retVal; + } + //endregion + //region Private Methods private static String buildUserAgent() { String retVal = null; diff --git a/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java b/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java index f5dbded5..5c6c9d7d 100644 --- a/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java +++ b/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java @@ -61,6 +61,40 @@ public static RestResponse sendPut(String url, String body, Map return retVal; } + + public static RestResponse sendDelete(String url, Map headers, + Map queryParams) throws ApiException { + + HttpDelete getRequest = new HttpDelete(url); + addQueryParams(getRequest, queryParams); + addHeaders(getRequest, headers); + RestResponse retVal = sendRequest(getRequest); + + return retVal; + } + + public static RestResponse sendPost(String url, String body, Map headers, + Map queryParams) throws ApiException { + + HttpPost postRequest = new HttpPost(url); + + if (body != null) { + StringEntity entity = null; + try { + entity = new StringEntity(body); + } + catch (UnsupportedEncodingException e) { + LOGGER.error("Exception when building put body", e); + } + postRequest.setEntity(entity); + } + + addQueryParams(postRequest, queryParams); + addHeaders(postRequest, headers); + RestResponse retVal = sendRequest(postRequest); + + return retVal; + } //endregion //region Private Methods diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 387080ac..66b2b05f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -18,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; @@ -32,7 +33,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud { private static final String CLOUD_URL = "aws/ec2"; protected Map executorsByInstanceType; private List executorsForTypes; - private List invalidInstanceTypes; + private List invalidInstanceTypes; //endregion //region Constructor @@ -117,7 +118,7 @@ public Boolean detachInstance(String instanceId) { } @Override - public void syncGroupInstances() { + protected void handleSyncGroupInstances() { IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); @@ -136,8 +137,6 @@ public void syncGroupInstances() { addNewSlaveInstances(instances); removeOldSlaveInstances(instances); - - } else { LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, @@ -145,12 +144,12 @@ public void syncGroupInstances() { } } - @Override - public Map getInstanceIpsById() { - Map retVal = new HashMap<>(); - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + @Override + public Map handleGetInstanceIpsById() { + Map retVal = new HashMap<>(); + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -208,7 +207,7 @@ protected int getOverridedNumberOfExecutors(String instanceType) { LOGGER.info(String.format("We have a weight definition for this type of %s", retVal)); } else { - retVal = NO_OVERRIDED_NUM_OF_EXECUTORS; + retVal = NO_OVERRIDDEN_NUM_OF_EXECUTORS; } return retVal; @@ -380,7 +379,7 @@ private void initExecutorsByInstanceType() { String type = instance.getAwsInstanceTypeFromAPIInput(); this.executorsByInstanceType.put(type, executors); - if(instance.getIsValid() == false){ + if (instance.getIsValid() == false) { LOGGER.error(String.format("Invalid type \'%s\' in group \'%s\'", type, this.getGroupId())); invalidInstanceTypes.add(type); } @@ -404,6 +403,7 @@ public List getInvalidInstanceTypes() { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot AWS Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index f7bf5a5d..4b496ee1 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -107,7 +108,7 @@ protected Integer getPendingThreshold() { } @Override - public void syncGroupInstances() { + protected void handleSyncGroupInstances() { IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); ApiResponse> instancesResponse = azureVmGroupRepo.getGroupVms(groupId, this.accountId); @@ -135,11 +136,11 @@ public void syncGroupInstances() { } @Override - public Map getInstanceIpsById() { + public Map handleGetInstanceIpsById() { Map retVal = new HashMap<>(); IAzureVmGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupVms(groupId, this.accountId); + ApiResponse> instancesResponse = awsGroupRepo.getGroupVms(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -193,7 +194,7 @@ private List handleNewVms(List newVms, S } private SpotinstSlave handleNewVm(String vmName, String vmSize, String label) { - Integer executors = getNumOfExecutors(vmSize); + Integer executors = getNumOfExecutors(vmSize); addToPending(vmName, executors, PendingInstance.StatusEnum.PENDING, label); SpotinstSlave retVal = buildSpotinstSlave(vmName, vmSize, String.valueOf(executors)); return retVal; @@ -260,8 +261,8 @@ private void addSpotinstSlave(AzureGroupVm vm) { SpotinstSlave slave = null; if (vm.getVmName() != null) { - String vmSize = vm.getVmSize(); - Integer executors = getNumOfExecutors(vmSize); + String vmSize = vm.getVmSize(); + Integer executors = getNumOfExecutors(vmSize); slave = buildSpotinstSlave(vm.getVmName(), vmSize, String.valueOf(executors)); } @@ -292,6 +293,7 @@ private Boolean isSlaveExistForInstance(AzureGroupVm vm) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot Azure Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index 99e25a99..c8160efd 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -21,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; @@ -73,7 +74,7 @@ public Boolean detachInstance(String instanceId) { Boolean retVal = false; IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); ApiResponse detachInstanceResponse = - azureGroupRepo.detachInstance(groupId, instanceId, this.accountId); + azureGroupRepo.detachInstance(groupId, instanceId, accountId); if (detachInstanceResponse.isRequestSucceed()) { LOGGER.info(String.format("Instance %s detached", instanceId)); @@ -93,7 +94,12 @@ public void syncGroupInstances() { } @Override - public Map getInstanceIpsById() { + protected void handleSyncGroupInstances() { + + } + + @Override + public Map getInstanceIpsById() {//TODO: check if is different Map retVal = new HashMap<>(); IAzureGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); @@ -104,7 +110,7 @@ public Map getInstanceIpsById() { List instances = instancesResponse.getValue(); for (AzureGroupInstance instance : instances) { - if (this.getShouldUsePrivateIp()) { + if (getShouldUsePrivateIp()) { retVal.put(instance.getInstanceId(), instance.getPrivateIp()); } else { @@ -121,6 +127,11 @@ public Map getInstanceIpsById() { return retVal; } + @Override + protected Map handleGetInstanceIpsById() { + return null; + } + @Override public void monitorInstances() { IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); @@ -322,6 +333,7 @@ private void addToGroupPending(ProvisionRequest request) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot Azure LPVM (old)"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 4071b270..6e55e94f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -1,11 +1,14 @@ package hudson.plugins.spotinst.cloud; -import edu.umd.cs.findbugs.annotations.CheckForNull; import hudson.DescriptorExtensionList; import hudson.model.*; import hudson.model.labels.LabelAtom; +import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.api.infra.JsonMapper; +import hudson.plugins.spotinst.cloud.helpers.TimeHelper; import hudson.plugins.spotinst.common.*; +import hudson.plugins.spotinst.repos.ILockRepo; +import hudson.plugins.spotinst.repos.RepoManager; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; import hudson.slaves.*; @@ -15,6 +18,7 @@ import hudson.tools.ToolLocationNodeProperty; import jenkins.model.Jenkins; import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.DataBoundSetter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +28,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.*; + /** * Created by ohadmuchnik on 25/05/2016. */ @@ -32,7 +38,7 @@ public abstract class BaseSpotinstCloud extends Cloud { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); - protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; + protected static final int NO_OVERRIDDEN_NUM_OF_EXECUTORS = -1; protected String accountId; protected String groupId; protected Map pendingInstances; @@ -110,6 +116,10 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat else { this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } + + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + syncGroupsOwner(); + } } //endregion @@ -119,28 +129,33 @@ public Collection provision(Label label, int excessWorkload) { ProvisionRequest request = new ProvisionRequest(label, excessWorkload); LOGGER.info(String.format("Got provision slave request: %s", JsonMapper.toJson(request))); + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); - setNumOfNeededExecutors(request); + if (isGroupManagedByThisController) { + setNumOfNeededExecutors(request); - if (request.getExecutors() > 0) { - LOGGER.info(String.format("Need to scale up %s units", request.getExecutors())); + if (request.getExecutors() > 0) { + LOGGER.info(String.format("Need to scale up %s units", request.getExecutors())); + List slaves = provisionSlaves(request); - List slaves = provisionSlaves(request); - - if (slaves.size() > 0) { - for (final SpotinstSlave slave : slaves) { - - try { - Jenkins.getInstance().addNode(slave); - } - catch (IOException e) { - LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), e); + if (slaves.size() > 0) { + for (final SpotinstSlave slave : slaves) { + try { + Jenkins.getInstance().addNode(slave); + } + catch (IOException e) { + LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), + e); + } } } } + else { + LOGGER.info("No need to scale up new slaves, there are some that are initiating"); + } } else { - LOGGER.info("No need to scale up new slaves, there are some that are initiating"); + handleGroupDoesNotManageByThisController(accountId, groupId); } return Collections.emptyList(); @@ -198,7 +213,7 @@ public void monitorInstances() { Integer pendingThreshold = getPendingThreshold(); Boolean isPendingOverThreshold = - TimeUtils.isTimePassed(pendingInstance.getCreatedAt(), pendingThreshold); + TimeUtils.isTimePassedInMinutes(pendingInstance.getCreatedAt(), pendingThreshold); if (isPendingOverThreshold) { LOGGER.info(String.format( @@ -329,7 +344,7 @@ protected void terminateOfflineSlaves(SpotinstSlave slave, String slaveInstanceI Date slaveCreatedAt = slave.getCreatedAt(); - Boolean isOverOfflineThreshold = TimeUtils.isTimePassed(slaveCreatedAt, offlineThreshold); + Boolean isOverOfflineThreshold = TimeUtils.isTimePassedInMinutes(slaveCreatedAt, offlineThreshold); if (isSlaveOffline && isSlaveConnecting == false && isOverOfflineThreshold && temporarilyOffline == false && isOverIdleThreshold) { @@ -347,14 +362,44 @@ public SlaveInstanceDetails getSlaveDetails(String instanceId) { return retVal; } + + // public boolean isCloudReadyForGroupCommunication(String groupId) { + // boolean retVal = false; + // + // if (StringUtils.isNotEmpty(groupId)) { + // SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); + // + // if (state != null) { + // if (state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { + // retVal = true; + // } + // } + // } + // + // return retVal; + // } + + public boolean isCloudReadyForGroupCommunication(String groupId) { + boolean retVal = false; + + if (StringUtils.isNotEmpty(groupId)) { + GroupStateTracker stateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (stateDetails != null) { + retVal = stateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY); + } + } + + return retVal; + } //endregion //region Private Methods private synchronized List provisionSlaves(ProvisionRequest request) { LOGGER.info(String.format("Scale up group: %s with %s workload units", groupId, request.getExecutors())); - List slaves = scaleUp(request); - return slaves; + List retVal = scaleUp(request); + return retVal; } private void setNumOfNeededExecutors(ProvisionRequest request) { @@ -434,6 +479,46 @@ private boolean isGlobalExecutorOverrideValid() { return retVal; } + + private Boolean LockGroupController(String controllerIdentifier) { + Boolean retVal = null; + ILockRepo lockRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse lockGroupControllerResponse = + lockRepo.Lock(groupId, accountId, controllerIdentifier, Constants.LOCK_TIME_TO_LIVE_IN_SECONDS); + + if (lockGroupControllerResponse.isRequestSucceed()) { + String responseValue = lockGroupControllerResponse.getValue(); + + if (Constants.LOCK_OK_STATUS.equals(responseValue)) { + LOGGER.info(String.format("Successfully locked group %s controller", groupId)); + + retVal = true; + } + else { + LOGGER.error(String.format("Failed locking group %s controller", groupId)); + retVal = false; + } + } + else { + LOGGER.error("lock request failed"); + } + + return retVal; + } + + // private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { + // SpotinstCloudCommunicationState state = null; + // + // for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() + // .getCloudsInitializationState() + // .entrySet()) { + // if (cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)) { + // state = cloudsInitializationStateEntry.getValue(); + // } + // } + // + // return state; + // } //endregion //region Protected Methods @@ -594,10 +679,10 @@ protected Integer getNumOfExecutors(String instanceType) { retVal = 1; } else { - int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); - boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDED_NUM_OF_EXECUTORS; + int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); + boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDDEN_NUM_OF_EXECUTORS; - if(isNumOfExecutorsOverrided){ + if (isNumOfExecutorsOverrided) { retVal = overridedNumOfExecutors; } else { @@ -633,7 +718,7 @@ protected Integer getNumOfExecutors(String instanceType) { } protected int getOverridedNumberOfExecutors(String instanceType) { - return NO_OVERRIDED_NUM_OF_EXECUTORS; + return NO_OVERRIDDEN_NUM_OF_EXECUTORS; } protected Integer getPendingThreshold() { @@ -643,6 +728,174 @@ protected Integer getPendingThreshold() { protected Integer getSlaveOfflineThreshold() { return Constants.SLAVE_OFFLINE_THRESHOLD_IN_MINUTES; } + + // public void handleGroupDosNotManageByThisController(String groupId) { + // boolean isGroupExistInCandidateGroupsForControllerOwnership = + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); + // + // if (isGroupExistInCandidateGroupsForControllerOwnership) { + // Boolean hasValidState = SpotinstContext.getInstance().getCloudsInitializationState().containsKey(this); + // + // if (hasValidState == false) { + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(this, SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + // } + // } + // else { + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + // } + // } + + public void handleGroupDoesNotManageByThisController(String accountId, String groupId) { + GroupStateTracker groupStateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (groupStateDetails == null || groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY)) { + groupStateDetails = new GroupStateTracker(groupId, accountId); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); + } + else if (groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING)) { + boolean shouldFail = TimeHelper.isTimePassedInSeconds(groupStateDetails.getTimeStamp()); + + if (shouldFail) { + groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_FAILED); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); + } + } + } + + // public void syncGroupsOwner(BaseSpotinstCloud cloud) { + // String groupId = cloud.getGroupId(); + // String accountId = cloud.getAccountId(); + // LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); + // + // ILockRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); + // String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + // + // if (redisGetValueResponse.isRequestSucceed()) { + // //redis response might return in different types + // if (redisGetValueResponse.getValue() instanceof String) { + // String redisResponseValue = (String) redisGetValueResponse.getValue(); + // + // if (redisResponseValue != null) { + // boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); + // + // if (isGroupBelongToController) { + // handleGroupManagedByThisController(cloud, controllerIdentifier); + // } + // else { + // LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, + // controllerIdentifier)); + // boolean isContainsCandidates = + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + // .containsKey(groupId); + // + // if (isContainsCandidates == false) { + // SpotinstCloudCommunicationState cloudState = + // SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); + // + // if (cloudState == null) { + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + // .put(groupId, accountId); + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(cloud, SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + // } + // } + // } + // } + // else { + // LOGGER.warn("redis response value return null"); + // } + // } + // //there is no controller for the given group in redis, should take ownership + // else { + // handleGroupManagedByThisController(cloud, controllerIdentifier); + // } + // } + // } + + public void syncGroupsOwner() { + ILockRepo lockRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse lockGroupControllerResponse = lockRepo.getLockValueById(groupId, accountId); + String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + + if (lockGroupControllerResponse.isRequestSucceed()) { + String lockGroupControllerValue = lockGroupControllerResponse.getValue(); + boolean isGroupBelongToNone = lockGroupControllerValue == null; + + if (isGroupBelongToNone) { + handleGroupManagedByNone(controllerIdentifier); + } + else { + boolean isGroupBelongToController = controllerIdentifier.equals(lockGroupControllerValue); + + if (isGroupBelongToController) { + handleGroupManagedByThisController(controllerIdentifier); + } + else { + LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, + controllerIdentifier)); + handleGroupDoesNotManageByThisController(accountId, groupId); + } + } + } + else { + LOGGER.error("group locking service failed to get lock for groupId {}, accountId {}.", groupId, accountId); + } + } + + private void handleGroupManagedByNone(String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + GroupStateTracker cloudDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (cloudDetails != null) { + SpotinstCloudCommunicationState cloudState = cloudDetails.getState(); + + if (cloudState.equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false) { + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); + } + } + else { + cloudDetails = new GroupStateTracker(groupId, accountId); + } + + Boolean hasLock = LockGroupController(controllerIdentifier); + + if (hasLock != null) { + if (hasLock) { + cloudDetails.setState(SPOTINST_CLOUD_COMMUNICATION_READY); + } + else { + cloudDetails = new GroupStateTracker(groupId, accountId); + } + } + + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, cloudDetails); + } + + private void handleGroupManagedByThisController(String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + GroupStateTracker cloudDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (cloudDetails != null) { + boolean isCloudNotReady = cloudDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false; + + if (isCloudNotReady) { + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); + } + } + + cloudDetails = new GroupStateTracker(groupId, accountId); + cloudDetails.setState(SPOTINST_CLOUD_COMMUNICATION_READY); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, cloudDetails); + //expand TTL of the current controller in redis + boolean isLockRevive = LockGroupController(controllerIdentifier); + + if (isLockRevive == false) { + LOGGER.warn("could not expand lock ttl for group {}", groupId); + } + } //endregion //region Getters / Setters @@ -773,13 +1026,13 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { // if enabled, enable and override GlobalExecutorOverride to 1 // better clarity to user, avoid race conditions - boolean shouldDisableGlobalExecutors = isSingleTaskNodesEnabled != null && isSingleTaskNodesEnabled && this.globalExecutorOverride != null; + boolean shouldDisableGlobalExecutors = + isSingleTaskNodesEnabled != null && isSingleTaskNodesEnabled && this.globalExecutorOverride != null; if (shouldDisableGlobalExecutors) { this.globalExecutorOverride.setIsEnabled(false); } } - //endregion //region Abstract Methods @@ -789,9 +1042,35 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { public abstract String getCloudUrl(); - public abstract void syncGroupInstances(); + public void syncGroupInstances() { + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + handleSyncGroupInstances(); + } + else { + handleGroupDoesNotManageByThisController(accountId, groupId); + } + } + + protected abstract void handleSyncGroupInstances(); + + public Map getInstanceIpsById() { + Map retVal; + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + retVal = handleGetInstanceIpsById(); + } + else { + handleGroupDoesNotManageByThisController(accountId, groupId); + retVal = new HashMap<>(); + } + + return retVal; + } - public abstract Map getInstanceIpsById(); + protected abstract Map handleGetInstanceIpsById(); protected abstract Integer getDefaultExecutorsNumber(String instanceType); //endregion diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 773a22b8..af53400c 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; @@ -115,7 +116,7 @@ public Boolean detachInstance(String instanceId) { } @Override - public void syncGroupInstances() { + protected void handleSyncGroupInstances() { IGcpGroupRepo gcpGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); ApiResponse> instancesResponse = gcpGroupRepo.getGroupInstances(groupId, this.accountId); @@ -144,11 +145,11 @@ public void syncGroupInstances() { } @Override - public Map getInstanceIpsById() { + public Map handleGetInstanceIpsById() { Map retVal = new HashMap<>(); IGcpGroupRepo awsGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -290,6 +291,7 @@ private void addSpotinstGcpSlave(GcpGroupInstance instance) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot GCP Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java b/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java new file mode 100644 index 00000000..fc774b8c --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java @@ -0,0 +1,59 @@ +package hudson.plugins.spotinst.cloud; + +import hudson.Extension; +import hudson.Plugin; +import hudson.model.Describable; +import hudson.model.Descriptor; +import jenkins.model.Jenkins; + +import java.io.IOException; +import java.util.logging.Logger; + +@Extension +public class PluginImpl extends Plugin implements Describable { + private static final Logger LOGGER = Logger.getLogger(PluginImpl.class.getName()); + + // Whether the SshHostKeyVerificationAdministrativeMonitor should show messages when we have templates using + // accept-new or check-new-soft strategies + private long dismissInsecureMessages; + + public void saveDismissInsecureMessages(long dismissInsecureMessages) { + this.dismissInsecureMessages = dismissInsecureMessages; + try { + save(); + } catch (IOException io) { + LOGGER.warning("There was a problem saving that you want to dismiss all messages related to insecure EC2 templates"); + } + } + + public long getDismissInsecureMessages() { + return dismissInsecureMessages; + } + + public DescriptorImpl getDescriptor() { + return (DescriptorImpl) Jenkins.get().getDescriptorOrDie(getClass()); + } + + public static PluginImpl get() { + return Jenkins.get().getPlugin(PluginImpl.class); + } + + @Extension + public static final class DescriptorImpl extends Descriptor { + @Override + public String getDisplayName() { + return "Spotinst PluginImpl"; + } + } + + @Override + public void postInitialize() throws IOException { + // backward compatibility with the legacy class name +// Jenkins.XSTREAM.alias("hudson.plugins.ec2.EC2Cloud", AmazonEC2Cloud.class); +// Jenkins.XSTREAM.alias("hudson.plugins.ec2.EC2Slave", EC2OndemandSlave.class); +// // backward compatibility with the legacy instance type +// Jenkins.XSTREAM.registerConverter(new InstanceTypeConverter()); + + load(); + } +} diff --git a/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java b/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java new file mode 100644 index 00000000..7c19ff88 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java @@ -0,0 +1,42 @@ +package hudson.plugins.spotinst.cloud.helpers; + +import java.util.Calendar; +import java.util.Date; + +public class TimeHelper { + //region members + private static final Integer redisTimeToLeaveInSeconds = 60 * 3; + private static final Integer miliToSeconds = 1000; + private static final Integer MILI_TO_SECONDS = 1000; + public static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = generateSuspendedGroupFetchingTime(); + //endregion + + //region methods + public static Boolean isTimePassedInSeconds(Date from) { + Boolean retVal = false; + Date now = new Date(); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(from); + calendar.add(Calendar.SECOND, SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS / MILI_TO_SECONDS); + Date timeToPass = calendar.getTime(); + + if (now.after(timeToPass)) { + retVal = true; + } + + return retVal; + } + + public static Integer getRedisTimeToLeaveInSeconds() { + return redisTimeToLeaveInSeconds; + } + //endregion + + //region private methods + private static Integer generateSuspendedGroupFetchingTime() { + Integer retVal = miliToSeconds * redisTimeToLeaveInSeconds + 10; + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java new file mode 100644 index 00000000..347a234b --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java @@ -0,0 +1,127 @@ +package hudson.plugins.spotinst.cloud.monitor; + +import hudson.Extension; +import hudson.model.AdministrativeMonitor; +import hudson.plugins.spotinst.common.GroupStateTracker; +import hudson.plugins.spotinst.common.SpotinstCloudCommunicationState; +import hudson.plugins.spotinst.common.SpotinstContext; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.*; + +@Extension +public class SpotinstCloudsCommunicationMonitor extends AdministrativeMonitor { + + //region Members + List spotinstCloudsCommunicationFailures; + List spotinstCloudsCommunicationInitializing; + //endregion + + //region Overridden Public Methods + @Override + public boolean isActivated() { + return isSpotinstCloudsCommunicationFailuresExist() || isSpotinstCloudsCommunicationInitializingExist(); + } + + @Override + public String getDisplayName() { + return "Spotinst Clouds Communication Monitor"; + } + //endregion + + //region getters & setters +// public boolean isSpotinstCloudsCommunicationFailuresExist() { +// boolean isCloudsWithFailureStateExist = SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue( +// SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); +// boolean isCloudsWithGroupIdExist = CollectionUtils.isNotEmpty(getGroupsIdByCloudInitializationState( +// SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED)); +// return isCloudsWithFailureStateExist && isCloudsWithGroupIdExist; +// } + + public boolean isSpotinstCloudsCommunicationFailuresExist() { + return isSpotinstCloudsCommunicationStateExist(SPOTINST_CLOUD_COMMUNICATION_FAILED); + } + +// public boolean isSpotinstCloudsCommunicationInitializingExist() { +// return SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); +// } + + public boolean isSpotinstCloudsCommunicationInitializingExist() { + return isSpotinstCloudsCommunicationStateExist(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + } + +// public boolean isSpotinstCloudsCommunicationReadyExist() { +// return SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); +// } + + private boolean isSpotinstCloudsCommunicationStateExist(SpotinstCloudCommunicationState state){ + Collection allCurrentGroups = + SpotinstContext.getInstance().getConnectionStateByGroupId().values(); + + boolean isCloudsWithReadyStateExist = allCurrentGroups.stream().anyMatch( + groupDetails -> state.equals( + groupDetails.getState())); + + return isCloudsWithReadyStateExist; + } + + public String getSpotinstCloudsCommunicationFailures() { + String retVal; + + spotinstCloudsCommunicationFailures = getGroupsIdByCloudInitializationState( + SPOTINST_CLOUD_COMMUNICATION_FAILED); + retVal = String.join(", ", spotinstCloudsCommunicationFailures); + + return retVal; + } + + public String getSpotinstCloudsCommunicationInitializing() { + String retVal; + + spotinstCloudsCommunicationInitializing = getGroupsIdByCloudInitializationState( + SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + retVal = String.join(", ", spotinstCloudsCommunicationInitializing); + + return retVal; + } + +// private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { +// List retVal = new ArrayList<>(); +// +// for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() +// .getCloudsInitializationState() +// .entrySet()) { +// if (cloudsInitializationStateEntry.getValue().equals(state)) { +// BaseSpotinstCloud cloud = cloudsInitializationStateEntry.getKey(); +// +// if (StringUtils.isNotEmpty(cloud.getGroupId())) { +// retVal.add(cloud.getGroupId()); +// } +// } +// } +// +// return retVal; +// } + + private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { + List retVal = new ArrayList<>(); + + Collection allCurrentGroups = + SpotinstContext.getInstance().getConnectionStateByGroupId().values(); + allCurrentGroups.forEach(group -> { + if(state.equals(group.getState()) && StringUtils.isNotEmpty(group.getGroupId())){ + retVal.add(group.getGroupId()); + } + }); + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/common/Constants.java b/src/main/java/hudson/plugins/spotinst/common/Constants.java index 9693d635..bcf70270 100644 --- a/src/main/java/hudson/plugins/spotinst/common/Constants.java +++ b/src/main/java/hudson/plugins/spotinst/common/Constants.java @@ -10,4 +10,6 @@ public class Constants { public static final Integer REST_CLIENT_CONNECT_TIMEOUT_IN_SECONDS = 120; public static final Integer REST_CLIENT_CONNECTION_REQUEST_TIMEOUT_IN_SECONDS = 120; public static final Integer REST_CLIENT_SOCKET_TIMEOUT_IN_SECONDS = 120; + public static final String LOCK_OK_STATUS = "OK"; + public static final Integer LOCK_TIME_TO_LIVE_IN_SECONDS = 60 * 3; } diff --git a/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java new file mode 100644 index 00000000..0c0f80e6 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java @@ -0,0 +1,45 @@ +package hudson.plugins.spotinst.common; + +import java.util.Date; + +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; + +public class GroupStateTracker { + //region members + private final String groupId; + private final String accountId; + private SpotinstCloudCommunicationState state; + private final Date timeStamp; + //endregion + + //region Constructor + public GroupStateTracker(String groupId, String accountId) { + this.groupId = groupId; + this.accountId = accountId; + state = SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; + timeStamp = new Date(); + } + //endregion + + //region getters & setters + public String getGroupId() { + return groupId; + } + + public String getAccountId() { + return accountId; + } + + public SpotinstCloudCommunicationState getState() { + return state; + } + + public void setState(SpotinstCloudCommunicationState state) { + this.state = state; + } + + public Date getTimeStamp() { + return timeStamp; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java new file mode 100644 index 00000000..bbb387cd --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java @@ -0,0 +1,41 @@ +package hudson.plugins.spotinst.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public enum SpotinstCloudCommunicationState { + SPOTINST_CLOUD_COMMUNICATION_INITIALIZING("SPOTINST CLOUD COMMUNICATION INITIALIZING"), + SPOTINST_CLOUD_COMMUNICATION_FAILED("SPOTINST CLOUD COMMUNICATION FAILED"), + SPOTINST_CLOUD_COMMUNICATION_READY("SPOTINST CLOUD COMMUNICATION READY"); + + // region members + private static final Logger LOGGER = LoggerFactory.getLogger(CredentialsMethodEnum.class); + + private String name; + // endregion + + SpotinstCloudCommunicationState(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public SpotinstCloudCommunicationState fromName(String name){ + SpotinstCloudCommunicationState retVal = null; + + for (SpotinstCloudCommunicationState enumValues : SpotinstCloudCommunicationState.values()) { + if (enumValues.name.equals(name)) { + retVal = enumValues; + break; + } + } + + if (retVal == null) { + LOGGER.warn(String.format( + "Tried to create CredentialsMethodEnum for name: %s, but we don't support such type ", name)); + } + return retVal; + } +} diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index 9a059e11..2f9e6f96 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -2,9 +2,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.plugins.spotinst.model.aws.AwsInstanceType; +import org.apache.commons.lang.RandomStringUtils; -import java.util.Date; -import java.util.List; +import java.util.*; /** * Created by ohadmuchnik on 24/05/2016. @@ -12,11 +12,15 @@ public class SpotinstContext { //region Members - private static SpotinstContext instance; - private String spotinstToken; - private String accountId; - private List awsInstanceTypes; - private Date awsInstanceTypesLastUpdate; + private static SpotinstContext instance; + private String spotinstToken; + private String accountId; + private List awsInstanceTypes; + private Date awsInstanceTypesLastUpdate; + private String controllerIdentifier; + private Map connectionStateByGroupId; +// private PassiveExpiringMap candidateGroupsForControllerOwnership; +// private Map cloudsInitializationState; //endregion public static SpotinstContext getInstance() { @@ -63,6 +67,38 @@ public Date getAwsInstanceTypesLastUpdate() { public void setAwsInstanceTypesLastUpdate(Date awsInstanceTypesLastUpdate) { this.awsInstanceTypesLastUpdate = awsInstanceTypesLastUpdate; } + + public String getControllerIdentifier() {//TODO: verify with Ziv + if(controllerIdentifier == null){ + controllerIdentifier = RandomStringUtils.randomAlphanumeric(10); + } + + return controllerIdentifier; + } + +// public PassiveExpiringMap getCandidateGroupsForControllerOwnership() { +// if (candidateGroupsForControllerOwnership == null) { +// candidateGroupsForControllerOwnership = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); +// } +// +// return candidateGroupsForControllerOwnership; +// } +// +// public Map getCloudsInitializationState() { +// if (cloudsInitializationState == null) { +// cloudsInitializationState = new HashMap<>(); +// } +// +// return cloudsInitializationState; +// } + + public Map getConnectionStateByGroupId() { + if(connectionStateByGroupId == null){ + connectionStateByGroupId = new HashMap<>(); + } + + return connectionStateByGroupId; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java new file mode 100644 index 00000000..a0d9012c --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java @@ -0,0 +1,52 @@ +package hudson.plugins.spotinst.common; + +import hudson.model.RestartListener; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.jobs.SpotinstSyncGroupsOwner; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +//TODO Liron - this part not working yet +public class SpotinstRestartListener extends RestartListener { + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstRestartListener.class); + + //TODO Liron - check if necessary + public static SpotinstRestartListener getInstance() { + return Jenkins.get() + .getExtensionList(RestartListener.class) + .get(SpotinstRestartListener.class); + } + + //TODO Liron - check if necessary + public SpotinstRestartListener(){} + + @Override + public boolean isReadyToRestart() throws IOException, InterruptedException { + return true; + } + + @Override + public void onRestart() { + SpotinstSyncGroupsOwner groupsOwnerJob = new SpotinstSyncGroupsOwner(); + List cloudList = Jenkins.getInstance().clouds; + Set cloudSet = new HashSet<>(); + + if (cloudList != null && cloudList.size() > 0) { + for (Cloud cloud : cloudList) { + if (cloud instanceof BaseSpotinstCloud) { + cloudSet.add((BaseSpotinstCloud) cloud); + } + } + } + + LOGGER.info(String.format("deallocating %s Spotinst clouds", cloudSet.size())); + groupsOwnerJob.deallocateCloudsNoLongerInUse(cloudSet);//TODO: execute instead? + } +} diff --git a/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java b/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java index 5d7a8c92..5b1c5379 100644 --- a/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java +++ b/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java @@ -8,7 +8,7 @@ * Created by ohadmuchnik on 21/03/2017. */ public class TimeUtils { - public static Boolean isTimePassed(Date from, Integer minutes) { + public static Boolean isTimePassedInMinutes(Date from, Integer minutes) { Boolean retVal = false; Date now = new Date(); Calendar calendar = Calendar.getInstance(); diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java new file mode 100644 index 00000000..fc397c65 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -0,0 +1,152 @@ +package hudson.plugins.spotinst.jobs; + +import hudson.Extension; +import hudson.model.AsyncPeriodicWork; +import hudson.model.TaskListener; +import hudson.plugins.spotinst.api.infra.ApiResponse; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.cloud.helpers.TimeHelper; +import hudson.plugins.spotinst.common.GroupStateTracker; +import hudson.plugins.spotinst.common.SpotinstContext; +import hudson.plugins.spotinst.repos.ILockRepo; +import hudson.plugins.spotinst.repos.RepoManager; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * Created by ohadmuchnik on 25/05/2016. + */ +@Extension +public class SpotinstSyncGroupsOwner extends AsyncPeriodicWork { + + //region Members + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncGroupsOwner.class); + private static final Integer redisToJobRatio = 3; + public final Integer JOB_INTERVAL_IN_SECONDS; + + final long recurrencePeriod; + //endregion + + //region Constructor + public SpotinstSyncGroupsOwner() { + super("Sync Groups Owner"); + JOB_INTERVAL_IN_SECONDS = TimeHelper.getRedisTimeToLeaveInSeconds() / redisToJobRatio; + recurrencePeriod = TimeUnit.SECONDS.toMillis(JOB_INTERVAL_IN_SECONDS); + } + //endregion + + //region Public Methods + // @Override + // protected void execute(TaskListener taskListener) { + // synchronized (this) { + // List cloudList = Jenkins.getInstance().clouds; + // Set cloudsFromContext = + // SpotinstContext.getInstance().getCloudsInitializationState().keySet(); + // Set cloudsNoLongerExist = new HashSet<>(cloudsFromContext); + // + // if (cloudList != null && cloudList.size() > 0) { + // for (Cloud cloud : cloudList) { + // + // if (cloud instanceof BaseSpotinstCloud) { + // BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + // String groupId = spotinstCloud.getGroupId(); + // String accountId = spotinstCloud.getAccountId(); + // cloudsNoLongerExist.remove(spotinstCloud); + // + // if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + // spotinstCloud.syncGroupsOwner(spotinstCloud); + // } + // } + // } + // } + // + // deallocateGroupsNoLongerInUse(cloudsNoLongerExist); + // } + // } + + @Override + protected void execute(TaskListener taskListener) { + synchronized (this) { + List cloudList = Jenkins.getInstance().clouds; + Set groupsFromContext = SpotinstContext.getInstance().getConnectionStateByGroupId().keySet(); + Set groupsNoLongerExist = new HashSet<>(groupsFromContext); + + if (cloudList != null && cloudList.size() > 0) { + for (Cloud cloud : cloudList) { + + if (cloud instanceof BaseSpotinstCloud) { + BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + String groupId = spotinstCloud.getGroupId(); + String accountId = spotinstCloud.getAccountId(); + groupsNoLongerExist.remove(groupId); + + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + spotinstCloud.syncGroupsOwner(); + } + } + } + } + + deallocateGroupsNoLongerInUse(groupsNoLongerExist); + } + } + + // public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { + // for (BaseSpotinstCloud cloud : cloudsNoLongerExist) { + // String groupId = cloud.getGroupId(); + // String accountId = cloud.getAccountId(); + // SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); + // + // if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + // ILockRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); + // + // if (redisGetValueResponse.isRequestSucceed()) { + // LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); + // } + // else { + // LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + // } + // } + // } + // } + + public void deallocateCloudsNoLongerInUse(Set cloudsNoLongerExist) { + Set groupsNoLongerExist = + cloudsNoLongerExist.stream().map(BaseSpotinstCloud::getGroupId).collect(Collectors.toSet()); + deallocateGroupsNoLongerInUse(groupsNoLongerExist); + } + + private void deallocateGroupsNoLongerInUse(Set groupsNoLongerExist) { + for (String groupId : groupsNoLongerExist) { + GroupStateTracker groupDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + String accountId = groupDetails.getAccountId(); + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); + + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + ILockRepo redisRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse redisGetValueResponse = redisRepo.Unlock(groupId, accountId); + + if (redisGetValueResponse.isRequestSucceed()) { + LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); + } + else { + LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + } + } + } + } + + @Override + public long getRecurrencePeriod() { + return recurrencePeriod; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java index 75f1804d..6b6c858f 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java @@ -1,11 +1,14 @@ package hudson.plugins.spotinst.jobs; import hudson.Extension; +import hudson.ExtensionList; import hudson.model.AsyncPeriodicWork; +import hudson.model.PeriodicWork; import hudson.model.TaskListener; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; import hudson.slaves.Cloud; import jenkins.model.Jenkins; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +25,7 @@ public class SpotinstSyncInstances extends AsyncPeriodicWork { private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncInstances.class); public static final Integer JOB_INTERVAL_IN_MINUTES = 1; final long recurrencePeriod; + boolean isJobExecuted; //endregion //region Constructor @@ -40,6 +44,16 @@ protected void execute(TaskListener taskListener) { for (Cloud cloud : cloudList) { if (cloud instanceof BaseSpotinstCloud) { BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + + if(this.isJobExecuted == false) { + ExtensionList spotinstGroupsOwnerMonitorPeriodicWork = SpotinstSyncGroupsOwner.all(); + + if (CollectionUtils.isNotEmpty(spotinstGroupsOwnerMonitorPeriodicWork)) { + spotinstGroupsOwnerMonitorPeriodicWork.get(0).run(); + this.isJobExecuted = true; + } + } + try { spotinstCloud.syncGroupInstances(); } diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java new file mode 100644 index 00000000..f8482288 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class GetGroupControllerLockResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java new file mode 100644 index 00000000..ce11981f --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java @@ -0,0 +1,35 @@ +package hudson.plugins.spotinst.model.redis; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class LockGroupControllerRequest { + + //region Memebers + private final String groupId; + private final String controllerIdentifier; + private final Integer ttl; + //endregion + + //region Constructor + public LockGroupControllerRequest(String groupId, String controllerIdentifier, Integer ttl) { + this.groupId = groupId; + this.controllerIdentifier = controllerIdentifier; + this.ttl = ttl; + } + //endregion + + //region Getters & Setters + public String getGroupId() { + return groupId; + } + + public String getControllerIdentifier() { + return controllerIdentifier; + } + + public Integer getTtl() { + return ttl; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java new file mode 100644 index 00000000..53c4efd2 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class LockGroupControllerResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java new file mode 100644 index 00000000..ec148506 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class UnlockGroupControllerResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java b/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java new file mode 100644 index 00000000..73975659 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java @@ -0,0 +1,12 @@ +package hudson.plugins.spotinst.repos; + +import hudson.plugins.spotinst.api.infra.ApiResponse; + +public interface ILockRepo { + ApiResponse Lock(String groupId, String accountId, String controllerIdentifier, Integer ttl); + + ApiResponse getLockValueById(String groupId, String accountId); + + ApiResponse Unlock(String groupId, String accountId); + +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/LockRepo.java b/src/main/java/hudson/plugins/spotinst/repos/LockRepo.java new file mode 100644 index 00000000..87a4aa64 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/repos/LockRepo.java @@ -0,0 +1,59 @@ +package hudson.plugins.spotinst.repos; + +import hudson.plugins.spotinst.api.SpotinstApi; +import hudson.plugins.spotinst.api.infra.ApiException; +import hudson.plugins.spotinst.api.infra.ApiResponse; +import hudson.plugins.spotinst.api.infra.ExceptionHelper; + +public class LockRepo implements ILockRepo { + + @Override + public ApiResponse Lock(String groupId, String accountId, String controllerIdentifier, Integer ttl) { + ApiResponse retVal; + + try { + String lockResult = SpotinstApi.LockGroupController(groupId, accountId, controllerIdentifier, ttl); + + retVal = new ApiResponse<>(lockResult); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + + @Override + public ApiResponse getLockValueById(String groupId, String accountId) { + ApiResponse retVal; + + try { + String controllerIdentifier = SpotinstApi.getGroupLockValueById(groupId, accountId); + retVal = new ApiResponse<>(controllerIdentifier); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + + @Override + public ApiResponse Unlock(String groupId, String accountId) { + ApiResponse retVal; + + try { + Integer isKeyDeleted = SpotinstApi.UnlockGroupController(groupId, accountId); + + retVal = new ApiResponse<>(isKeyDeleted); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java index 2392347e..6d2a82fe 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java +++ b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java @@ -10,6 +10,7 @@ public class RepoManager { private IAzureGroupRepo azureGroupRepo; private IAzureVmGroupRepo azureVmGroupRepo; private IAwsInstanceTypesRepo awsInstanceTypesRepo; + private ILockRepo lockRepo; //endregion //region Constructor @@ -19,6 +20,7 @@ private RepoManager() { this.azureGroupRepo = new AzureGroupRepo(); this.azureVmGroupRepo = new AzureVmGroupRepo(); this.awsInstanceTypesRepo = new AwsInstanceTypesRepo(); + this.lockRepo = new LockRepo(); } private static RepoManager instance = new RepoManager(); @@ -68,5 +70,13 @@ public IAwsInstanceTypesRepo getAwsInstanceTypesRepo() { public void setAwsInstanceTypesRepo(IAwsInstanceTypesRepo awsInstanceTypesRepo) { this.awsInstanceTypesRepo = awsInstanceTypesRepo; } + + public ILockRepo getLockRepo() { + return lockRepo; + } + + public void setLockRepo(ILockRepo lockRepo) { + this.lockRepo = lockRepo; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 37cb2ee2..8c34a1b8 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -25,8 +25,6 @@ public class SpotinstComputer extends SlaveComputer { //endregion //region overrides - // better than JobListener because in complicated pipelines Jenkins is the initial Node and - // we know the Node is a SpotinstNode (I'm not using that word). @Override public void taskAccepted(Executor executor, Queue.Task task) { super.taskAccepted(executor, task); @@ -42,8 +40,6 @@ public void taskAccepted(Executor executor, Queue.Task task) { spotinstNode.getNodeName(), spotinstCloud.getDisplayName()); LOGGER.info(msg); this.setAcceptingTasks(false); - // I see much better responsiveness from Jenkins in case of setting offline, both have same effect - // should also be decided with UI because each combo looks different in the node list. SpotinstNonLocalizable spotinstNonLocalizable = new SpotinstNonLocalizable(msg); SpotinstSingleTaskOfflineCause spotinstSingleTaskOfflineCause = new SpotinstSingleTaskOfflineCause(spotinstNonLocalizable); this.setTemporarilyOffline(true,spotinstSingleTaskOfflineCause); diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 51c556c4..c52457d6 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Date; import java.util.List; @@ -173,42 +174,63 @@ public Node asNode() { //region Public Methods public void terminate() { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); - - if (isTerminated) { - LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); - removeIfInPending(); - try { - Jenkins.getInstance().removeNode(this); - } - catch (IOException e) { - e.printStackTrace(); + String groupId = getSpotinstCloud().getGroupId(); + String accountId = getSpotinstCloud().getAccountId(); + boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + + if (isTerminated) { + LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); + removeIfInPending(); + try { + Jenkins.getInstance().removeNode(this); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); } } - else { - LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); - } + else{ + try { + getSpotinstCloud().handleGroupDoesNotManageByThisController(accountId, groupId); + }catch (Exception e){ + LOGGER.warn(e.getMessage()); + } } } public Boolean forceTerminate() { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + Boolean retVal = false; - if (isTerminated) { - LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); - removeIfInPending(); - } - else { - LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); - } + String groupId = getSpotinstCloud().getGroupId(); + String accountId = getSpotinstCloud().getAccountId(); + boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); - try { - Jenkins.get().removeNode(this); + if (isTerminated) { + LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); + removeIfInPending(); + } else { + LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); + } + + try { + Jenkins.get().removeNode(this); + } catch (IOException e) { + e.printStackTrace(); + } + + retVal = isTerminated; } - catch (IOException e) { - e.printStackTrace(); + else{ + getSpotinstCloud().handleGroupDoesNotManageByThisController(accountId, groupId); } - return isTerminated; + return retVal; } private void removeIfInPending() { @@ -235,6 +257,7 @@ public void onSlaveConnected() { @Extension public static class DescriptorImpl extends SlaveDescriptor { + @Nonnull @Override public String getDisplayName() { return "Spot Node"; diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly new file mode 100644 index 00000000..47caa25b --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly @@ -0,0 +1,37 @@ + + +
+ + ${%Title} +

+ + ${%SpotinstCloudsCommunicationFailures(it.spotinstCloudsCommunicationFailures)} +

+ + ${%Explanation(rootURL)} +

+ +

+ +
+ + ${%Title} +

+ + ${%SpotinstCloudsCommunicationInitializing(it.spotinstCloudsCommunicationInitializing)} +

+ +

+ +
+ + ${%Title} +

+ + rootURL + + ${%SpotinstCloudsCommunicationReady(it.spotinstCloudsCommunicationReady)} +

+ +

+ diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties new file mode 100644 index 00000000..06e30210 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties @@ -0,0 +1,5 @@ +Title=Spotinst Clouds Communication Monitor +SpotinstCloudsCommunicationInitializing=Communication of Spotinst clouds with the following Elastigroups is initializing: {0} +SpotinstCloudsCommunicationReady=Spotinst clouds with the following Elastigroups are ready for communication: {0} +SpotinstCloudsCommunicationFailures=The following Elastigroups are already connected to a different Jenkins controller: {0} +Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. \ No newline at end of file