cloudstack.go

package cloudstack

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/state"
	"github.com/xanzy/go-cloudstack/cloudstack"
)

const (
	driverName      = "cloudstack"
	dockerPort      = 2376
    kubePort        = 6443
    etcdPortStart   = 2379
    etcdPortEnd     = 2380
    webPort   = 80
	swarmPort       = 3376
	diskDatadisk    = "DATADISK"
)

type configError struct {
	option string
}

func (e *configError) Error() string {
	return fmt.Sprintf("cloudstack driver requires the --cloudstack-%s option", e.option)
}

type Driver struct {
	*drivers.BaseDriver
	Id                   string
	ApiURL               string
	ApiKey               string
	SecretKey            string
	HTTPGETOnly          bool
	JobTimeOut           int64
	UsePrivateIP         bool
	UsePortForward       bool
	PublicIP             string
	PublicIPID           string
	DisassociatePublicIP bool
	SSHKeyPair           string
	PrivateIP            string
	CIDRList             []string
	FirewallRuleIds      []string
	Expunge              bool
	Template             string
	TemplateID           string
	ServiceOffering      string
	ServiceOfferingID    string
	DeleteVolumes        bool
	DiskOffering         string
	DiskOfferingID       string
	DiskSize             int
	Network              string
	NetworkID            string
	Zone                 string
	ZoneID               string
	NetworkType          string
	UserDataFile         string
	UserData             string
	Project              string
	ProjectID            string
	Tags                 []string
	DisplayName          string
}

// GetCreateFlags registers the flags this driver adds to
// "docker hosts create"
func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	return []mcnflag.Flag{
		mcnflag.StringFlag{
			Name:   "cloudstack-api-url",
			Usage:  "CloudStack API URL",
			EnvVar: "CLOUDSTACK_API_URL",
		},
		mcnflag.StringFlag{
			Name:   "cloudstack-api-key",
			Usage:  "CloudStack API key",
			EnvVar: "CLOUDSTACK_API_KEY",
		},
		mcnflag.StringFlag{
			Name:   "cloudstack-secret-key",
			Usage:  "CloudStack API secret key",
			EnvVar: "CLOUDSTACK_SECRET_KEY",
		},
		mcnflag.BoolFlag{
			Name:   "cloudstack-http-get-only",
			Usage:  "Only use HTTP GET to execute CloudStack API",
			EnvVar: "CLOUDSTACK_HTTP_GET_ONLY",
		},
		mcnflag.IntFlag{
			Name:   "cloudstack-timeout",
			Usage:  "time(seconds) allowed to complete async job",
			EnvVar: "CLOUDSTACK_TIMEOUT",
			Value:  300,
		},
		mcnflag.BoolFlag{
			Name:  "cloudstack-use-private-address",
			Usage: "Use a private IP to access the machine",
		},
		mcnflag.BoolFlag{
			Name:  "cloudstack-use-port-forward",
			Usage: "Use port forwarding rule to access the machine",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-public-ip",
			Usage: "CloudStack Public IP",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-ssh-user",
			Usage: "CloudStack SSH user",
			Value: "root",
		},
		mcnflag.StringSliceFlag{
			Name:  "cloudstack-cidr",
			Usage: "Source CIDR to give access to the machine. default 0.0.0.0/0",
		},
		mcnflag.BoolFlag{
			Name:  "cloudstack-expunge",
			Usage: "Whether or not to expunge the machine upon removal",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-template",
			Usage: "CloudStack template",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-template-id",
			Usage: "Cloudstack template id",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-service-offering",
			Usage: "CloudStack service offering",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-service-offering-id",
			Usage: "CloudStack service offering id",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-network",
			Usage: "CloudStack network",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-network-id",
			Usage: "CloudStack network id",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-zone",
			Usage: "CloudStack zone",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-zone-id",
			Usage: "CloudStack zone id",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-userdata-file",
			Usage: "CloudStack Userdata file",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-project",
			Usage: "CloudStack project",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-project-id",
			Usage: "CloudStack project id",
		},
		mcnflag.StringSliceFlag{
			Name:  "cloudstack-resource-tag",
			Usage: "key:value resource tags to be created",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-disk-offering",
			Usage: "Cloudstack disk offering",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-disk-offering-id",
			Usage: "Cloudstack disk offering id",
		},
		mcnflag.IntFlag{
			Name:  "cloudstack-disk-size",
			Usage: "Disk offering custom size",
		},
		mcnflag.BoolFlag{
			Name:  "cloudstack-delete-volumes",
			Usage: "Whether or not to delete data volumes associated with the machine upon removal",
		},
		mcnflag.StringFlag{
			Name:  "cloudstack-displayname",
			Usage: "Cloudstack virtual machine displayname",
		},
	}
}

func NewDriver(hostName, storePath string) drivers.Driver {
	driver := &Driver{
		BaseDriver: &drivers.BaseDriver{
			MachineName: hostName,
			StorePath:   storePath,
		},
		FirewallRuleIds: []string{},
	}
	return driver
}

// DriverName returns the name of the driver as it is registered
func (d *Driver) DriverName() string {
	return driverName
}

func (d *Driver) GetSSHHostname() (string, error) {
	return d.GetIP()
}

func (d *Driver) GetSSHUsername() string {
	if d.SSHUser == "" {
		d.SSHUser = "root"
	}
	return d.SSHUser
}

// SetConfigFromFlags configures the driver with the object that was returned
// by RegisterCreateFlags
func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error {
	d.ApiURL = flags.String("cloudstack-api-url")
	d.ApiKey = flags.String("cloudstack-api-key")
	d.SecretKey = flags.String("cloudstack-secret-key")
	d.UsePrivateIP = flags.Bool("cloudstack-use-private-address")
	d.UsePortForward = flags.Bool("cloudstack-use-port-forward")
	d.HTTPGETOnly = flags.Bool("cloudstack-http-get-only")
	d.JobTimeOut = int64(flags.Int("cloudstack-timeout"))
	d.SSHUser = flags.String("cloudstack-ssh-user")
	d.CIDRList = flags.StringSlice("cloudstack-cidr")
	d.Expunge = flags.Bool("cloudstack-expunge")
	d.Tags = flags.StringSlice("cloudstack-resource-tag")
	d.DeleteVolumes = flags.Bool("cloudstack-delete-volumes")
	d.DiskSize = flags.Int("cloudstack-disk-size")
	d.DisplayName = flags.String("cloudstack-displayname")
	d.SwarmMaster = flags.Bool("swarm-master")
	d.SwarmDiscovery = flags.String("swarm-discovery")
	if err := d.setProject(flags.String("cloudstack-project"), flags.String("cloudstack-project-id")); err != nil {
		return err
	}
	if err := d.setZone(flags.String("cloudstack-zone"), flags.String("cloudstack-zone-id")); err != nil {
		return err
	}
	if err := d.setTemplate(flags.String("cloudstack-template"), flags.String("cloudstack-template-id")); err != nil {
		return err
	}
	if err := d.setServiceOffering(flags.String("cloudstack-service-offering"), flags.String("cloudstack-service-offering-id")); err != nil {
		return err
	}
	if err := d.setNetwork(flags.String("cloudstack-network"), flags.String("cloudstack-network-id")); err != nil {
		return err
	}
	if err := d.setPublicIP(flags.String("cloudstack-public-ip")); err != nil {
		return err
	}
	if err := d.setUserData(flags.String("cloudstack-userdata-file")); err != nil {
		return err
	}
	if err := d.setDiskOffering(flags.String("cloudstack-disk-offering"), flags.String("cloudstack-disk-offering-id")); err != nil {
		return err
	}
	if d.DisplayName == "" {
		d.DisplayName = d.MachineName
	}
	d.SSHKeyPair = d.MachineName
	if d.ApiURL == "" {
		return &configError{option: "api-url"}
	}
	if d.ApiKey == "" {
		return &configError{option: "api-key"}
	}
	if d.SecretKey == "" {
		return &configError{option: "secret-key"}
	}
	if d.Template == "" {
		return &configError{option: "template"}
	}
	if d.ServiceOffering == "" {
		return &configError{option: "service-offering"}
	}
	if d.Zone == "" {
		return &configError{option: "zone"}
	}
	if len(d.CIDRList) == 0 {
		d.CIDRList = []string{"0.0.0.0/0"}
	}
	d.DisassociatePublicIP = false
	return nil
}

// GetURL returns a Docker compatible host URL for connecting to this host
// e.g. tcp://1.2.3.4:2376
func (d *Driver) GetURL() (string, error) {
	ip, err := d.GetIP()
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s:%d", ip, dockerPort), nil
}

// GetIP returns the IP that this host is available at
func (d *Driver) GetIP() (string, error) {
	if d.UsePrivateIP {
		return d.PrivateIP, nil
	}
	return d.PublicIP, nil
}

// GetState returns the state that the host is in (running, stopped, etc)
func (d *Driver) GetState() (state.State, error) {
	cs := d.getClient()
	vm, count, err := cs.VirtualMachine.GetVirtualMachineByID(d.Id, d.setParams)
	if err != nil {
		return state.Error, err
	}

	if count == 0 {
		return state.None, fmt.Errorf("Machine does not exist, use create command to create it")
	}

	switch vm.State {
	case "Starting":
		return state.Starting, nil
	case "Running":
		return state.Running, nil
	case "Stopping":
		return state.Running, nil
	case "Stopped":
		return state.Stopped, nil
	case "Destroyed":
		return state.Stopped, nil
	case "Expunging":
		return state.Stopped, nil
	case "Migrating":
		return state.Paused, nil
	case "Error":
		return state.Error, nil
	case "Unknown":
		return state.Error, nil
	case "Shutdowned":
		return state.Stopped, nil
	}

	return state.None, nil
}

// PreCreate allows for pre-create operations to make sure a driver is ready for creation
func (d *Driver) PreCreateCheck() error {

	if err := d.checkKeyPair(); err != nil {
		return err
	}

	if err := d.checkInstance(); err != nil {
		return err
	}

	return nil
}

// Create a host using the driver's config
func (d *Driver) Create() error {
	cs := d.getClient()
	if err := d.createKeyPair(); err != nil {
		return err
	}
	p := cs.VirtualMachine.NewDeployVirtualMachineParams(
		d.ServiceOfferingID, d.TemplateID, d.ZoneID)
	p.SetName(d.MachineName)
	p.SetDisplayname(d.DisplayName)
	p.SetKeypair(d.SSHKeyPair)
	if d.UserData != "" {
		p.SetUserdata(d.UserData)
	}
	if d.NetworkID != "" {
		p.SetNetworkids([]string{d.NetworkID})
	}
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	if d.DiskOfferingID != "" {
		p.SetDiskofferingid(d.DiskOfferingID)
		if d.DiskSize != 0 {
			p.SetSize(int64(d.DiskSize))
		}
	}
	if d.NetworkType == "Basic" {
		if err := d.createSecurityGroup(); err != nil {
			return err
		}
		p.SetSecuritygroupnames([]string{d.MachineName})
	}
	log.Info("Creating CloudStack instance ...")
	vm, err := cs.VirtualMachine.DeployVirtualMachine(p)
	if err != nil {
		return err
	}
	d.Id = vm.Id
	d.PrivateIP = vm.Nic[0].Ipaddress
	if d.NetworkType == "Basic" {
		d.PublicIP = d.PrivateIP
	}
	if d.NetworkType == "Advanced" && !d.UsePrivateIP {
		if d.PublicIPID == "" {
			if err := d.associatePublicIP(); err != nil {
				return err
			}
		}
		if err := d.configureFirewallRules(); err != nil {
			return err
		}
		if d.UsePortForward {
			if err := d.configurePortForwardingRules(); err != nil {
				return err
			}
		} else {
			if err := d.enableStaticNat(); err != nil {
				return err
			}
		}
	}
	if len(d.Tags) > 0 {
		if err := d.createTags(); err != nil {
			return err
		}
	}
	return nil
}

// Remove a host
func (d *Driver) Remove() error {
	cs := d.getClient()
	p := cs.VirtualMachine.NewDestroyVirtualMachineParams(d.Id)
	p.SetExpunge(d.Expunge)
	if err := d.deleteFirewallRules(); err != nil {
		return err
	}
	if err := d.disassociatePublicIP(); err != nil {
		return err
	}
	if err := d.deleteKeyPair(); err != nil {
		return err
	}
	log.Info("Removing CloudStack instance...")
	if _, err := cs.VirtualMachine.DestroyVirtualMachine(p); err != nil {
		return err
	}
	if d.NetworkType == "Basic" {
		if err := d.deleteSecurityGroup(); err != nil {
			return err
		}
	}
	if d.DeleteVolumes {
		if err := d.deleteVolumes(); err != nil {
			return err
		}
	}
	return nil
}

// Start a host
func (d *Driver) Start() error {
	vmstate, err := d.GetState()
	if err != nil {
		return err
	}

	if vmstate == state.Running {
		log.Info("Machine is already running")
		return nil
	}

	if vmstate == state.Starting {
		log.Info("Machine is already starting")
		return nil
	}

	cs := d.getClient()
	p := cs.VirtualMachine.NewStartVirtualMachineParams(d.Id)

	if _, err = cs.VirtualMachine.StartVirtualMachine(p); err != nil {
		return err
	}

	return nil
}

// Stop a host gracefully
func (d *Driver) Stop() error {
	vmstate, err := d.GetState()
	if err != nil {
		return err
	}

	if vmstate == state.Stopped {
		log.Info("Machine is already stopped")
		return nil
	}

	cs := d.getClient()
	p := cs.VirtualMachine.NewStopVirtualMachineParams(d.Id)

	if _, err = cs.VirtualMachine.StopVirtualMachine(p); err != nil {
		return err
	}

	return nil
}

// Restart a host.
func (d *Driver) Restart() error {
	vmstate, err := d.GetState()
	if err != nil {
		return err
	}

	if vmstate == state.Stopped {
		return fmt.Errorf("Machine is stopped, use start command to start it")
	}

	cs := d.getClient()
	p := cs.VirtualMachine.NewRebootVirtualMachineParams(d.Id)

	if _, err = cs.VirtualMachine.RebootVirtualMachine(p); err != nil {
		return err
	}

	return nil
}

// Kill stops a host forcefully
func (d *Driver) Kill() error {
	return d.Stop()
}

func (d *Driver) getClient() *cloudstack.CloudStackClient {
	cs := cloudstack.NewAsyncClient(d.ApiURL, d.ApiKey, d.SecretKey, false)
	cs.HTTPGETOnly = d.HTTPGETOnly
	cs.AsyncTimeout(d.JobTimeOut)
	return cs
}

func (d *Driver) setZone(zone string, zoneID string) error {
	d.Zone = zone
	d.ZoneID = zoneID
	d.NetworkType = ""

	if d.Zone == "" && d.ZoneID == "" {
		return nil
	}

	cs := d.getClient()

	var z *cloudstack.Zone
	var err error
	if d.ZoneID != "" {
		z, _, err = cs.Zone.GetZoneByID(d.ZoneID, d.setParams)
	} else {
		z, _, err = cs.Zone.GetZoneByName(d.Zone, d.setParams)
	}
	if err != nil {
		return fmt.Errorf("Unable to get zone: %v", err)
	}

	d.Zone = z.Name
	d.ZoneID = z.Id
	d.NetworkType = z.Networktype

	log.Debugf("zone: %q", d.Zone)
	log.Debugf("zone id: %q", d.ZoneID)
	log.Debugf("network type: %q", d.NetworkType)

	return nil
}

func (d *Driver) setTemplate(templateName string, templateID string) error {
	d.Template = templateName
	d.TemplateID = templateID

	if d.Template == "" && d.TemplateID == "" {
		return nil
	}

	if d.ZoneID == "" {
		return fmt.Errorf("Unable to get template: zone is not set")
	}

	cs := d.getClient()
	var template *cloudstack.Template
	var err error
	if d.TemplateID != "" {
		template, _, err = cs.Template.GetTemplateByID(d.TemplateID, "executable", d.setParams)
	} else {
		template, _, err = cs.Template.GetTemplateByName(d.Template, "executable", d.ZoneID, d.setParams)
	}
	if err != nil {
		return fmt.Errorf("Unable to get template: %v", err)
	}

	d.TemplateID = template.Id
	d.Template = template.Name

	log.Debugf("template id: %q", d.TemplateID)
	log.Debugf("template name: %q", d.Template)

	return nil
}

func (d *Driver) setServiceOffering(serviceoffering string, serviceofferingID string) error {
	d.ServiceOffering = serviceoffering
	d.ServiceOfferingID = serviceofferingID

	if d.ServiceOffering == "" && d.ServiceOfferingID == "" {
		return nil
	}

	cs := d.getClient()
	var service *cloudstack.ServiceOffering
	var err error
	if d.ServiceOfferingID != "" {
		service, _, err = cs.ServiceOffering.GetServiceOfferingByID(d.ServiceOfferingID, d.setParams)
	} else {
		service, _, err = cs.ServiceOffering.GetServiceOfferingByName(d.ServiceOffering, d.setParams)
	}
	if err != nil {
		return fmt.Errorf("Unable to get service offering: %v", err)
	}

	d.ServiceOfferingID = service.Id
	d.ServiceOffering = service.Name

	log.Debugf("service offering id: %q", d.ServiceOfferingID)
	log.Debugf("service offering name: %q", d.ServiceOffering)

	return nil
}

func (d *Driver) setDiskOffering(diskOffering string, diskOfferingID string) error {
	d.DiskOffering = diskOffering
	d.DiskOfferingID = diskOfferingID

	if d.DiskOffering == "" && d.DiskOfferingID == "" {
		return nil
	}

	cs := d.getClient()
	var disk *cloudstack.DiskOffering
	var err error
	if d.DiskOfferingID != "" {
		disk, _, err = cs.DiskOffering.GetDiskOfferingByID(d.DiskOfferingID, d.setParams)
	} else {
		disk, _, err = cs.DiskOffering.GetDiskOfferingByName(d.DiskOffering, d.setParams)
	}
	if err != nil {
		return fmt.Errorf("Unable to get disk offering: %v", err)
	}

	d.DiskOfferingID = disk.Id
	d.DiskOffering = disk.Name

	log.Debugf("disk offering id: %q", d.DiskOfferingID)
	log.Debugf("disk offering name: %q", d.DiskOffering)

	return nil
}

func (d *Driver) setNetwork(networkName string, networkID string) error {
	d.Network = networkName
	d.NetworkID = networkID

	if d.Network == "" && d.NetworkID == "" {
		return nil
	}

	cs := d.getClient()
	var network *cloudstack.Network
	var err error
	if d.NetworkID != "" {
		network, _, err = cs.Network.GetNetworkByID(d.NetworkID, d.setParams)
	} else {
		network, _, err = cs.Network.GetNetworkByName(d.Network, d.setParams)
	}
	if err != nil {
		return fmt.Errorf("Unable to get network: %v", err)
	}

	d.NetworkID = network.Id
	d.Network = network.Name

	log.Debugf("network id: %q", d.NetworkID)
	log.Debugf("network name: %q", d.Network)

	return nil
}

func (d *Driver) setPublicIP(publicip string) error {
	d.PublicIP = publicip
	d.PublicIPID = ""

	if d.PublicIP == "" {
		return nil
	}

	cs := d.getClient()
	p := cs.Address.NewListPublicIpAddressesParams()
	p.SetIpaddress(d.PublicIP)
	ips, err := cs.Address.ListPublicIpAddresses(p)
	if err != nil {
		return fmt.Errorf("Unable to get public ip id: %s", err)
	}
	if ips.Count < 1 {
		return fmt.Errorf("Unable to get public ip id: Not Found %s", d.PublicIP)
	}

	d.PublicIPID = ips.PublicIpAddresses[0].Id

	log.Debugf("public ip id: %q", d.PublicIPID)

	return nil
}

func (d *Driver) setUserData(userDataFile string) error {
	d.UserDataFile = userDataFile

	if d.UserDataFile == "" {
		return nil
	}

	data, err := ioutil.ReadFile(d.UserDataFile)
	if err != nil {
		return fmt.Errorf("Failed to read user data file: %s", err)
	}

	d.UserData = base64.StdEncoding.EncodeToString(data)

	return nil
}

func (d *Driver) setProject(projectName string, projectID string) error {
	d.Project = projectName
	d.ProjectID = projectID

	if d.Project == "" && d.ProjectID == "" {
		return nil
	}

	cs := d.getClient()
	var p *cloudstack.Project
	var err error
	if d.ProjectID != "" {
		p, _, err = cs.Project.GetProjectByID(d.ProjectID)
	} else {
		p, _, err = cs.Project.GetProjectByName(d.Project)
	}
	if err != nil {
		return fmt.Errorf("Invalid project: %s", err)
	}

	d.ProjectID = p.Id
	d.Project = p.Name

	log.Debugf("project id: %s", d.ProjectID)
	log.Debugf("project name: %s", d.Project)

	return nil
}

func (d *Driver) checkKeyPair() error {
	cs := d.getClient()

	log.Infof("Checking if SSH key pair (%v) already exists...", d.SSHKeyPair)

	p := cs.SSH.NewListSSHKeyPairsParams()
	p.SetName(d.SSHKeyPair)
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	res, err := cs.SSH.ListSSHKeyPairs(p)
	if err != nil {
		return err
	}
	if res.Count > 0 {
		return fmt.Errorf("SSH key pair (%v) already exists.", d.SSHKeyPair)
	}
	return nil
}

func (d *Driver) checkInstance() error {
	cs := d.getClient()

	log.Infof("Checking if instance (%v) already exists...", d.MachineName)

	p := cs.VirtualMachine.NewListVirtualMachinesParams()
	p.SetName(d.MachineName)
	p.SetZoneid(d.ZoneID)
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	res, err := cs.VirtualMachine.ListVirtualMachines(p)
	if err != nil {
		return err
	}
	if res.Count > 0 {
		return fmt.Errorf("Instance (%v) already exists.", d.SSHKeyPair)
	}
	return nil
}

func (d *Driver) createKeyPair() error {
	cs := d.getClient()

	if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return err
	}

	publicKey, err := ioutil.ReadFile(d.GetSSHKeyPath() + ".pub")
	if err != nil {
		return err
	}

	log.Infof("Registering SSH key pair...")

	p := cs.SSH.NewRegisterSSHKeyPairParams(d.SSHKeyPair, string(publicKey))
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	if _, err := cs.SSH.RegisterSSHKeyPair(p); err != nil {
		return err
	}

	return nil
}

func (d *Driver) deleteKeyPair() error {
	cs := d.getClient()

	log.Infof("Deleting SSH key pair...")

	p := cs.SSH.NewDeleteSSHKeyPairParams(d.SSHKeyPair)
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	if _, err := cs.SSH.DeleteSSHKeyPair(p); err != nil {
		return err
	}
	return nil
}

func (d *Driver) deleteVolumes() error {
	cs := d.getClient()

	log.Info("Deleting volumes...")

	p := cs.Volume.NewListVolumesParams()
	p.SetVirtualmachineid(d.Id)
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	volResponse, err := cs.Volume.ListVolumes(p)
	if err != nil {
		return err
	}
	for _, v := range volResponse.Volumes {
		if v.Type != diskDatadisk {
			continue
		}
		p := cs.Volume.NewDetachVolumeParams()
		p.SetId(v.Id)
		_, err := cs.Volume.DetachVolume(p)
		if err != nil {
			return err
		}
		_, err = cs.Volume.DeleteVolume(cs.Volume.NewDeleteVolumeParams(v.Id))
		if err != nil {
			return err
		}
	}
	return nil
}

func (d *Driver) associatePublicIP() error {
	cs := d.getClient()
	log.Infof("Associating public ip address...")
	p := cs.Address.NewAssociateIpAddressParams()
	p.SetZoneid(d.ZoneID)
	if d.NetworkID != "" {
		p.SetNetworkid(d.NetworkID)
	}
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	ip, err := cs.Address.AssociateIpAddress(p)
	if err != nil {
		return err
	}
	d.PublicIP = ip.Ipaddress
	d.PublicIPID = ip.Id
	d.DisassociatePublicIP = true

	return nil
}

func (d *Driver) disassociatePublicIP() error {
	if !d.DisassociatePublicIP {
		return nil
	}

	cs := d.getClient()
	log.Infof("Disassociating public ip address...")
	p := cs.Address.NewDisassociateIpAddressParams(d.PublicIPID)
	if _, err := cs.Address.DisassociateIpAddress(p); err != nil {
		return err
	}

	return nil
}

func (d *Driver) enableStaticNat() error {
	cs := d.getClient()
	log.Infof("Enabling Static Nat...")
	p := cs.NAT.NewEnableStaticNatParams(d.PublicIPID, d.Id)
	if _, err := cs.NAT.EnableStaticNat(p); err != nil {
		return err
	}

	return nil
}

func (d *Driver) configureFirewallRule(publicPort, privatePort int) error {
	cs := d.getClient()

	log.Debugf("Creating firewall rule ... : cidr list: %v, port %d", d.CIDRList, publicPort)
	p := cs.Firewall.NewCreateFirewallRuleParams(d.PublicIPID, "tcp")
	p.SetCidrlist(d.CIDRList)
	p.SetStartport(publicPort)
	p.SetEndport(publicPort)
	rule, err := cs.Firewall.CreateFirewallRule(p)
	if err != nil {
		// If the error reports the port is already open, just ignore.
		if !strings.Contains(err.Error(), fmt.Sprintf(
			"The range specified, %d-%d, conflicts with rule", publicPort, publicPort)) {
			return err
		}
	} else {
		d.FirewallRuleIds = append(d.FirewallRuleIds, rule.Id)
	}

	return nil
}

func (d *Driver) configurePortForwardingRule(publicPort, privatePort int) error {
	cs := d.getClient()

	log.Debugf("Creating port forwarding rule ... : cidr list: %v, port %d", d.CIDRList, publicPort)
	p := cs.Firewall.NewCreatePortForwardingRuleParams(
		d.PublicIPID, privatePort, "tcp", publicPort, d.Id)
	p.SetOpenfirewall(false)
	if _, err := cs.Firewall.CreatePortForwardingRule(p); err != nil {
		return err
	}

	return nil
}

func (d *Driver) configureFirewallRules() error {
	log.Info("Creating firewall rule for ssh port ...")

	if err := d.configureFirewallRule(22, 22); err != nil {
		return err
	}

	log.Info("Creating firewall rule for docker port ...")
	if err := d.configureFirewallRule(dockerPort, dockerPort); err != nil {
		return err
	}

	log.Info("Creating firewall rule for Kubernetes port ...")
	if err := d.configureFirewallRule(kubePort, kubePort); err != nil {
		return err
	}

	log.Info("Creating firewall rules for etcd ports ...")
	if err := d.configureFirewallRule(etcdPortStart, etcdPortStart); err != nil {
		return err
	}
	if err := d.configureFirewallRule(etcdPortEnd, etcdPortEnd); err != nil {
		return err
	}

    log.Info("Creating firewall rule for web port ...")
	if err := d.configureFirewallRule(webPort, webPort); err != nil {
		return err
	}

	if d.SwarmMaster {
		log.Info("Creating firewall rule for swarm port ...")
		if err := d.configureFirewallRule(swarmPort, swarmPort); err != nil {
			return err
		}
	}

	return nil
}

func (d *Driver) deleteFirewallRules() error {
	if len(d.FirewallRuleIds) > 0 {
		log.Info("Removing firewall rules...")
		for _, id := range d.FirewallRuleIds {
			cs := d.getClient()
			f := cs.Firewall.NewDeleteFirewallRuleParams(id)
			if _, err := cs.Firewall.DeleteFirewallRule(f); err != nil {
				return err
			}
		}
	}
	return nil
}

func (d *Driver) configurePortForwardingRules() error {
	log.Info("Creating port forwarding rule for ssh port ...")

	if err := d.configurePortForwardingRule(22, 22); err != nil {
		return err
	}

	log.Info("Creating port forwarding rule for docker port ...")
	if err := d.configurePortForwardingRule(dockerPort, dockerPort); err != nil {
		return err
	}

	log.Info("Creating port forwarding rule for Kubernetes port ...")
	if err := d.configurePortForwardingRule(kubePort, kubePort); err != nil {
		return err
	}

	log.Info("Creating port forwarding rules for etcd ports ...")
	if err := d.configurePortForwardingRule(etcdPortStart, etcdPortStart); err != nil {
		return err
	}
	if err := d.configurePortForwardingRule(etcdPortEnd, etcdPortEnd); err != nil {
		return err
	}

	log.Info("Creating port forwarding rule for web port ...")
	if err := d.configurePortForwardingRule(webPort, webPort); err != nil {
		return err
	}

	if d.SwarmMaster {
		log.Info("Creating port forwarding rule for swarm port ...")
		if err := d.configurePortForwardingRule(swarmPort, swarmPort); err != nil {
			return err
		}
	}

	return nil
}

func (d *Driver) createSecurityGroup() error {
	log.Debugf("Creating security group ...")
	cs := d.getClient()

	p1 := cs.SecurityGroup.NewCreateSecurityGroupParams(d.MachineName)
	if d.ProjectID != "" {
		p1.SetProjectid(d.ProjectID)
	}
	if _, err := cs.SecurityGroup.CreateSecurityGroup(p1); err != nil {
		return err
	}

	p2 := cs.SecurityGroup.NewAuthorizeSecurityGroupIngressParams()
	p2.SetSecuritygroupname(d.MachineName)
	p2.SetProtocol("tcp")
	p2.SetCidrlist(d.CIDRList)

	p2.SetStartport(22)
	p2.SetEndport(22)
	if d.ProjectID != "" {
		p2.SetProjectid(d.ProjectID)
	}
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	p2.SetStartport(dockerPort)
	p2.SetEndport(dockerPort)
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	p2.SetStartport(kubePort)
	p2.SetEndport(kubePort)
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	p2.SetStartport(etcdPortStart)
	p2.SetEndport(etcdPortStart)
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	p2.SetStartport(etcdPortEnd)
	p2.SetEndport(etcdPortEnd)
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	p2.SetStartport(webPort)
	p2.SetEndport(webPort)
	if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
		return err
	}

	if d.SwarmMaster {
		p2.SetStartport(swarmPort)
		p2.SetEndport(swarmPort)
		if _, err := cs.SecurityGroup.AuthorizeSecurityGroupIngress(p2); err != nil {
			return err
		}
	}
	return nil
}

func (d *Driver) deleteSecurityGroup() error {
	log.Debugf("Deleting security group ...")
	cs := d.getClient()

	p := cs.SecurityGroup.NewDeleteSecurityGroupParams()
	p.SetName(d.MachineName)
	if d.ProjectID != "" {
		p.SetProjectid(d.ProjectID)
	}
	if _, err := cs.SecurityGroup.DeleteSecurityGroup(p); err != nil {
		return err
	}
	return nil
}

func (d *Driver) createTags() error {
	log.Info("Creating resource tags ...")
	cs := d.getClient()
	tags := make(map[string]string)
	for _, t := range d.Tags {
		parts := strings.SplitN(t, ":", 2)
		if len(parts) != 2 {
			return fmt.Errorf("invalid resource tags format, each tag must be on the format KEY:VALUE")
		}
		tags[parts[0]] = parts[1]
	}
	params := cs.Resourcetags.NewCreateTagsParams([]string{d.Id}, "UserVm", tags)
	_, err := cs.Resourcetags.CreateTags(params)
	return err
}

func (d *Driver) setParams(c *cloudstack.CloudStackClient, p interface{}) error {
	if o, ok := p.(interface {
		SetProjectid(string)
	}); ok && d.ProjectID != "" {
		o.SetProjectid(d.ProjectID)
	}
	if o, ok := p.(interface {
		SetZoneid(string)
	}); ok && d.ZoneID != "" {
		o.SetZoneid(d.ZoneID)
	}
	return nil
}