diff --git a/docs/network.rst b/docs/network.rst new file mode 100644 index 0000000..b94378a --- /dev/null +++ b/docs/network.rst @@ -0,0 +1,5 @@ +:mod:`smartdc.network` Module +============================= + +.. autoclass:: smartdc.network.Network + diff --git a/docs/tef.rst b/docs/tef.rst new file mode 100644 index 0000000..29f8934 --- /dev/null +++ b/docs/tef.rst @@ -0,0 +1,5 @@ +:mod:`smartdc.tef` Module +============================ + +.. autoclass:: smartdc.tef.TefDataCenter + diff --git a/smartdc/__init__.py b/smartdc/__init__.py index 4ccbc5a..8fe31b8 100644 --- a/smartdc/__init__.py +++ b/smartdc/__init__.py @@ -1,6 +1,8 @@ from .datacenter import * from .machine import * from .legacy import LegacyDataCenter +from .network import * +from .tef import * from ._version import get_versions __version__ = get_versions()['version'] diff --git a/smartdc/datacenter.py b/smartdc/datacenter.py index e8aef6e..1ff4e5a 100644 --- a/smartdc/datacenter.py +++ b/smartdc/datacenter.py @@ -15,8 +15,7 @@ __version__ = get_versions()['version'] del get_versions -__all__ = ['DataCenter', 'KNOWN_LOCATIONS', - 'TELEFONICA_LOCATIONS', 'DEFAULT_LOCATION'] +__all__ = ['DataCenter', 'KNOWN_LOCATIONS', 'DEFAULT_LOCATION'] API_HOST_SUFFIX = '.api.joyentcloud.com' @@ -27,13 +26,6 @@ u'eu-ams-1': u'https://eu-ams-1.api.joyentcloud.com', } -TELEFONICA_LOCATIONS = { - u'London': u'https://api-eu-lon-1.instantservers.telefonica.com', - u'eu-lon-1': u'https://api-eu-lon-1.instantservers.telefonica.com', - u'Madrid': u'https://api-eu-mad-1.instantservers.telefonica.com', - u'eu-mad-1': u'https://api-eu-mad-1.instantservers.telefonica.com', -} - DEFAULT_LOCATION = 'us-west-1' DEFAULT_HEADERS = { @@ -154,7 +146,7 @@ def __repr__(self): user_string = '<{0}> '.format(self.login) else: user_string = '' - return '<{module}.{cls}: {name}at <{loc}>>'.format( + return '<{module}.{cls}: {name} at <{loc}>>'.format( module=self.__module__, cls=self.__class__.__name__, name=user_string, loc=self.location) @@ -236,7 +228,7 @@ def request(self, method, path, headers=None, data=None, **kwargs): if self.verbose: print("%s\t%s\t%s" % (datetime.now().isoformat(), method, full_path), - file=self.verbose) + file=sys.stderr) resp = requests.request(method, full_path, auth=self.auth, headers=request_headers, data=jdata, verify=self.verify, **kwargs) @@ -246,12 +238,14 @@ def request(self, method, path, headers=None, data=None, **kwargs): return self.request(method, path, headers=headers, data=data, **kwargs) if 400 <= resp.status_code < 499: - if resp.content: + if resp.content and self.verbose: print(resp.content, file=sys.stderr) resp.raise_for_status() if resp.content: - if resp.headers['content-type'] == 'application/json': - return (json.loads(resp.content), resp) + ctype = re.match('application/json(; +charset *= *([a-zA-z0-9-_]+))?', + resp.headers['content-type']) + if ctype: + return (json.loads(resp.content, ctype.groups('utf-8')[1]), resp) else: return (resp.content, resp) else: @@ -270,7 +264,7 @@ def api(self): if self.verbose: print("%s\t%s\t%s" % (datetime.now().isoformat(), 'GET', self.base_url), - file=self.verbose) + file=sys.stderr) resp = requests.request('GET', self.base_url, verify=self.verify) if 400 <= resp.status_code < 499: resp.raise_for_status() @@ -764,7 +758,8 @@ def create_machine(self, name=None, package=None, dataset=None, params['networks'] = [networks] j, r = self.request('POST', 'machines', data=params) if r.status_code >= 400: - print(j, file=sys.stderr) + if self.verbose: + print(j, file=sys.stderr) r.raise_for_status() return Machine(datacenter=self, data=j) @@ -884,4 +879,4 @@ def image(self, identifier): return j - \ No newline at end of file + diff --git a/smartdc/network.py b/smartdc/network.py new file mode 100644 index 0000000..c6d05b9 --- /dev/null +++ b/smartdc/network.py @@ -0,0 +1,403 @@ +import requests +import time +import uuid +import json +import re + +__all__ = ['Network'] + +class Network(object): + """ + A local proxy representing the state of a remote NetworkAPI subnet. + + A :py:class:`smartdc.network.Network` object is intended to be a + convenient container for methods and data relevant to a remotely running + subnet managed by NetworkAPI. A :py:class:`smartdc.network.Network` is + tied to a :py:class:`smartdc.tef.TefDataCenter` object, and makes all + its requests via that interface. It does not attempt to manage the state + cache in most cases, instead requiring the user to explicitly update with + a :py:meth:`refresh` call. + """ + def __init__(self, datacenter, network_id=None, data=None): + """ + :param datacenter: datacenter that contains this network + :type datacenter: :py:class:`smartdc.tef.TefDataCenter` + + :param network_id: unique ID of the network + :type network_id: :py:class:`basestring` + + :param data: raw data for instantiation + :type data: :py:class:`dict` + + Typically, a :py:class:`smartdc.network.Network` object is + instantiated automatically by a + :py:class:`smartdc.tef.TefDataCenter` object, but a user may + instantiate one with a minimum of a `datacenter` parameter and a + unique ID according to the network. The object then pulls in the + network data from the datacenter API. If `data` is passed in to + instantiate, then ingest the dict and populate internal values from + that. + + All of the following attributes are read-only: + + :var name: human-readable label for the network + :var id: identifier for the network + :var subnet: private subnet (CIDR) + :var resolver_ips: :py:class:`list` of DNS resolver IPs + :var private_gw_ip: private IP of the subnet gateway (:py:class:`basestring`) + :var public_gw_ip: public IP of the subnet gateway (:py:class:`basestring`) + :var status: last-known state of the network (:py:class:`basestring`) + """ + self.id = network_id or data.pop('id') + self.datacenter = datacenter + """the :py:class:`smartdc.tef.TefDataCenter` object that holds + this machine""" + if not data: + data = self.datacenter.raw_network_data(self.id) + self._save(data) + + def __str__(self): + """ + Represents the Network by its unique ID as a string. + """ + return self.id + + def __repr__(self): + """ + Presents a readable representation. + """ + if self.datacenter: + dc = str(self.datacenter) + else: + dc = '' + return '<{module}.{cls}: <{name}> in {dc}>'.format( + module=self.__module__, cls=self.__class__.__name__, + name=self.name, dc=dc) + + def __eq__(self, other): + if isinstance(other, dict): + return self.id == other.get('id') + elif isinstance(other, Network): + return self.id == other.id + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return uuid.UUID(self.id).int + + def _save(self, data): + """ + Take the data from a dict and commit them to appropriate attributes. + """ + self.name = data.get('name') + self.subnet = data.get('subnet') + self.resolver_ips = data.get('resolver_ips') + self.private_gw_ip = data.get('private_gw_ip') + self.public_gw_ip = data.get('public_gw_ip') + self.state = data.get('status') + + @property + def path(self): + """ + Convenience property to insert the id into a relative path for + requests. + """ + return 'networks/{id}'.format(id=self.id) + + @classmethod + def create_in_datacenter(cls, datacenter, name, subnet, **kwargs): + """ + :: + + POST /:login/networks + + Class method, provided as a convenience. + + :param dataceter: datacenter for creating the network + :type datacenter: :py:class:`smartdc.tef.TefDataCenter` + + Provision a network in the current + :py:class:`smartdc.tef.TefDataCenter`, returning an + instantiated :py:class:`smartdc.network.Network` object. + + 'datacenter', 'name' and 'subnet' are required arguments. + The rest of them are passed to the datacenter object as with + :py:meth:`smartdc.tef.TefDataCenter.create_network`. + """ + return datacenter.create_network(name, subnet, **kwargs) + + def refresh(self): + """ + :: + + GET /:login/networks/:id + + Fetch the existing status and values for the + :py:class:`smartdc.network.Network` from the datacenter + and commit the values locally. + """ + data = self.datacenter.raw_network_data(self.id) + self._save(data) + + def status(self): + """ + :: + + GET /:login/networks/:id + + :Returns: the current network status + :rtype: :py:class:`basestring` + + Refresh the network's information by fetching it remotely, then + returning the :py:attr:`state` as a string. + """ + self.refresh() + return self.state + + def delete(self): + """ + :: + + DELETE /:login/machines/:id + + Initiate deletion of an empty network. + """ + j, r = self.datacenter.request('DELETE', self.path) + r.raise_for_status() + + def poll_until(self, status, interval=2): + """ + :: + + GET /:login/networks/:id + + :param status: target status + :type status: :py:class:`basestring` + + :param interal: pause in seconds between polls + :type interval: :py:class:`int` + + Convenience method that continuously polls the current state of the + machine remotely, and returns until the named `status` argument is + reached. The default wait `interval` between requests is 2 seconds, + but it may be changed. + + .. Note:: If the next status is wrongly identified, this method may + loop forever. + """ + while self.status() != status: + time.sleep(interval) + + def poll_while(self, status, interval=2): + """ + :: + + GET /:login/networks/:id + + :param status: (assumed) current status + :type status: :py:class:`basestring` + + :param interval: pause in seconds between polls + :type interval: :py:class:`int` + + Convenience method that continuously polls the current status of the + network remotely, and returns while the network has the named `status` + argument. Once the status changes, the method returns. The default wait + `interval` between requests is 2 seconds, but it may be changed. + + .. Note:: If a status transition has not correctly been triggered, this + method may loop forever. + """ + while self.status() == status: + time.sleep(interval) + + def set_outbound_status(self, enabled): + """ + :: + + PUT /:login/networks/:id/outbound + + :param enabled: new status for the current network outbound PAT (Port + Address Translation). + :type enabled: :py:class:`bool` + + :Returns: the updated network outbound PAT (Port Address Translation) + status. + :rtype: :py:class:`bool` + """ + assert isinstance(enabled, bool), "Illegal status" + j, r = self.datacenter.request('PUT', self.path + '/outbound', + data={ 'enabled': enabled }) + r.raise_for_status() + return j['enabled'] + + def get_outbound_status(self): + """ + :: + + GET /:login/networks/:id/outbound + + :Returns: the current network outbound PAT (Port Address Translation) + status. + :rtype: :py:class:`bool` + """ + j, r = self.datacenter.request('GET', self.path + '/outbound') + r.raise_for_status() + return j['enabled'] + + def get_inbound_rules(self): + """ + :: + + GET /:login/networks/:id/inbound + + :Returns: a list containing all the inbound port forwarding + rules for the current Netowrk. + :rtype: :py:class:`list` of :py:class:`dict` + """ + j, _ = self.datacenter.request('GET', self.path + '/inbound') + return j + + def add_inbound_rule(self, name, start_port, destination_ip, end_port=None, + protocols=None, source_subnet=None, destination_base_port=None): + """ + :: + + PUT /:login/networks/:id/inbound + + :param name: the name of the new inbound port forwarding rule + to add. Up to 32 letters, digits and hyphens. Required. + :type name: :py:class:`basestring` + + :param start_port: first inbound port to forward according to this + rule (0...65535). Required. + :type start_port: :py:class:`int` + + :param end_port: last inbound port to forward according to this rule + (0...65535). If ommitted, the `start_port` will be taken. + :type end_port: :py:class:`int` + + :param protocols: list of protocols to forward according to this list. + Can be a list of strings or a single value. If omitted, both TCP + and UPD will be forwarded. + :type protocols: :py:class:`list` or :py:class:`basestring` + + :param source_subnet: a CIDR specifying the origin of the traffic to + be forwarded. If omitted, it will be set to '0.0.0.0/0', thus + forwarding all incoming traffic. + :type source_subnet: :py:class:`basestring` + + :param destination_ip: internal IP to which the inbound traffic + accepted by this rule should be forwarded to. Required. + :type destination_ip: :py:class:`basestring` + + :param destination_base_port: (first) port of the destination IP to + forward the inbound traffic to (0...65535). If omitted, the + `start_port` will be taken. + :type destination_base_port: :py:class:`int` + + :Returns: a dict containing the new inbound port forwarding rule. + :rtype: :py:class:`dict` + """ + params = {} + assert re.match(r'[a-zA-Z0-9-_]{1,32}', name), "Illegal name" + params['name'] = name + assert start_port>=0 and start_port<=65535, "Illegal start_port" + params['start_port'] = start_port + if end_port: + assert end_port>=start_port and end_port<=65535, \ + "Illegal end_port" + else: + end_port = start_port + params['end_port'] = end_port + if protocols: + if isinstance(protocols, basestring): + protocols = [protocols] + else: + protocols = ["tcp", "udp"] + params['protocols'] = protocols + if source_subnet: + assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(\/[0-9]+)?', \ + source_subnet), "Illegal source_subnet" + if not source_subnet.find('/'): + source_subnet = source_subnet + "/32" + params['source_subnet'] = source_subnet + assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', destination_ip), \ + "Illegal destination_ip" + params['destination_ip'] = destination_ip + if destination_base_port: + assert destination_base_port>=0 and destination_base_port<=65535, \ + "Illegal destination_base_port" + else: + destination_base_port = start_port + params['destination_base_port'] = destination_base_port + j, r = self.datacenter.request('POST', self.path + '/inbound', data=params) + r.raise_for_status() + return j + + def set_inbound_rule_status(self, rule_id, enabled): + """ + :: + + PUT /:login/networks/:id/inbound/:rule_id + + :param rule_id: id of the inbound port forwarding rule. + :type rule_id: :py:class:`basestring` + :param enabled: new status for the former inbound port forwarding rule. + :type enabled: :py:class:`bool` + + :Returns: the updated inbound port forwarding rule status. + :rtype: :py:class:`bool` + """ + assert isinstance(rule_id, basestring), "Illegal rule_id" + assert isinstance(enabled, bool), "Illegal status" + j, r = self.datacenter.request('PUT', self.path + '/inbound/' + rule_id, + data={ 'enabled': enabled }) + r.raise_for_status() + return j['enabled'] + + def get_outbound_rule_status(self, rule_id): + """ + :: + + GET /:login/networks/:id/inbound/:rule_id + + :param rule_id: id of the inbound port forwarding rule. + :type rule_id: :py:class:`basestring` + + :Returns: the current status of the specified inbound port + forwarding rule. + :rtype: :py:class:`bool` + """ + assert isinstance(rule_id, basestring), "Illegal rule_id" + j, r = self.datacenter.request('GET', self.path + '/inbound/' + rule_id) + r.raise_for_status() + return j['enabled'] + + def delete_inbound_rule(self, rule_id): + """ + + DELETE /:login/networks/:id/inbound/:rule_id + + :param rule_id: id of the inbound port forwarding rule to delete. + :type rule_id: :py:class:`basestring` + """ + assert isinstance(rule_id, basestring), "Illegal rule_id" + _, r = self.datacenter.request('DELETE', self.path + '/inbound/' + rule_id) + r.raise_for_status() + + def delete_all_inbound_rules(self): + """ + + DELETE /:login/networks/:id/inbound/:rule_id + + Deletes all inbound port forwarding rules of this network. + """ + for rule in self.get_inbound_rules(): + _, r = self.datacenter.request('DELETE', self.path + '/inbound/' + \ + rule['id']) + r.raise_for_status() + diff --git a/smartdc/tef.py b/smartdc/tef.py new file mode 100644 index 0000000..b95f300 --- /dev/null +++ b/smartdc/tef.py @@ -0,0 +1,224 @@ +from __future__ import print_function +from .legacy import LegacyDataCenter +from .network import Network +from .machine import Machine + +import re +import sys + +__all__ = ['TefDataCenter', 'TELEFONICA_LOCATIONS', 'ACENS_LOCATIONS'] + +TELEFONICA_LOCATIONS = { + u'London': u'https://api-eu-lon-1.instantservers.telefonica.com', + u'eu-lon-1': u'https://api-eu-lon-1.instantservers.telefonica.com', + u'Madrid': u'https://api-eu-mad-1.instantservers.telefonica.com', + u'eu-mad-1': u'https://api-eu-mad-1.instantservers.telefonica.com', +} + +ACENS_LOCATIONS = { + u'London': u'https://api-lon.instantservers.es', + u'lon': u'https://api-lon.instantservers.es', + u'Madrid': u'https://api-mad.instantservers.es', + u'mad': u'https://api-mad.instantservers.es', +} + +class TefDataCenter(LegacyDataCenter): + """ + This class provides support for the network extensiones present + on TEF datacenters to the legacy (~6.5) SmartDataCenter API. + + More information: https://api-eu-lon-1.instantservers.telefonica.com/docs/networkapi_docs.html + """ + + def create_network(self, name, subnet, resolver_ips=None): + """ + :: + + POST /:login/networks + + Provision a machine in the current + :py:class:`smartdc.tef.TefDataCenter`, returning an instantiated + :py:class:`smartdc.network.Network` object. + + :param name: a human-readable label for the machine + :type name: :py:class:`basestring`, up to 32 letters, digits and + hyphens. This parameter value is required. + + :param subnet: a private :type subnet: :py:class:`basestring` (CDR), + containing a base IP address plus a mask, which must be in the + range from /22 to /27. This parameter value is required. + + :param resolver_ips: list of DNS resolver IPs to be used by the + subnet. If not supplied, the default IPs ("8.8.8.8" and "4.4.4.4") + will be used. + :type resolver_ips: :py:class:`list` + + :rtype: :py:class:`smartdc.network.Network' + """ + params = {} + assert re.match(r'[a-zA-Z0-9-]{1,32}', name), "Illegal name" + params['name'] = name + assert re.match(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\/[0-9]+', + subnet), "Illegal subnet" + params['subnet'] = subnet + if resolver_ips: + if isinstance(resolver_ips, list): + params['resolver_ips'] = resolver_ips + j, r = self.request('POST', 'networks', data=params) + if r.status_code >= 400: + if self.verbose: + print(j, file=sys.stderr) + r.raise_for_status() + return Network(datacenter=self, data=j) + + def raw_network_data(self, network_id): + """ + :: + + GET /:login/networks/:network + + :param network_id: identifier for the network + :type network_id: :py:class:`basestring` or :py:class:`dict` + + :rtype: :py:class:`dict` + + Primarily used internally to get a raw dict for a single network. + """ + params = {} + if isinstance(network_id, dict): + network_id = network_id['id'] + j, r = self.request('GET', 'networks/' + str(network_id), + params=params) + return j + + def networks(self, search=None, fields=('name,')): + """ + :: + + GET /:login/networks + + :param search: optionally filter (locally) with a regular expression + search on the listed fields + :type search: :py:class:`basestring` that compiles as a regular + expression + + :param fields: filter on the listed fields (defaulting to + ``name``) + :type fields: :py:class:`list` of :py:class:`basestring`\s + + :Returns: network available in this datacenter + :rtype: :py:class:`list` of :py:class:`dict`\s + """ + j, _ = self.request('GET', 'networks') + if search: + j = list(search_dicts(j, search, fields)) + return [Network(datacenter=self, data=m) for m in j] + + def network(self, identifier): + """ + :: + + GET /:login/networks/:id + + :param identifier: match on the listed network identifier + :type identifier: :py:class:`basestring` or :py:class:`dict` + + :Returns: characteristics of the requested network + :rtype: :py:class:`dict` + + Either a string or a dictionary with an ``id`` key may be passed in. + """ + + if isinstance(identifier, dict): + identifier = identifier.get('id') + j, _ = self.request('GET', 'networks/' + str(identifier)) + return j + + def create_machine(self, name=None, package=None, dataset=None, + metadata=None, tags=None, boot_script=None, credentials=False, + image=None, network_id=None): + """ + :: + + POST /:login/machines + + Provision a machine in the current + :py:class:`smartdc.tef.TefDataCenter`, returning an instantiated + :py:class:`smartdc.machine.Machine` object. All of the parameter + values are optional, as they are assigned default values by the + datacenter's API itself. + + :param name: a human-readable label for the machine + :type name: :py:class:`basestring` + + :param package: cluster of resource values identified by name + :type package: :py:class:`basestring` or :py:class:`dict` + + :param image: an identifier for the base operating system image + (formerly a ``dataset``) + :type image: :py:class:`basestring` or :py:class:`dict` + + :param dataset: base operating system image identified by a globally + unique ID or URN (deprecated) + :type dataset: :py:class:`basestring` or :py:class:`dict` + + :param metadata: keys & values with arbitrary supplementary + details for the machine, accessible from the machine itself + :type metadata: :py:class:`dict` + + :param tags: keys & values with arbitrary supplementary + identifying information for filtering when querying for machines + :type tags: :py:class:`dict` + + :param network_id: network id where this machine will belong to; if + omitted, the machine will have a public IP address. PLEASE note + that if this parameter is specified, then ``name``, ``package`` + and ``dataset`` are compulsory. + :type network_id: :py:class:`basestring` + + :param boot_script: path to a file to upload for execution on boot + :type boot_script: :py:class:`basestring` as file path + + :rtype: :py:class:`smartdc.machine.Machine` + + If `package`, `image`, or `dataset` are passed a :py:class:`dict` containing a + `name` key (in the case of `package`) or an `id` key (in the case of + `image` or `dataset`), it passes the corresponding value. The server API + appears to resolve incomplete or ambiguous dataset URNs with the + highest version number. + """ + params = {} + if name: + assert re.match(r'[a-zA-Z0-9]([a-zA-Z0-9\-\.]*[a-zA-Z0-9])?$', + name), "Illegal name" + params['name'] = name + if package: + if isinstance(package, dict): + package = package['name'] + params['package'] = package + if image: + if isinstance(image, dict): + image = image['id'] + params['image'] = image + if dataset and not image: + if isinstance(dataset, dict): + dataset = dataset.get('id', dataset['urn']) + params['dataset'] = dataset + if metadata: + for k, v in metadata.items(): + params['metadata.' + str(k)] = v + if tags: + for k, v in tags.items(): + params['tag.' + str(k)] = v + if boot_script: + with open(boot_script) as f: + params['metadata.user-script'] = f.read() + if network_id: + if isinstance(network_id, basestring): + params['network_id'] = network_id + j, r = self.request('POST', 'machines', data=params) + if r.status_code >= 400: + if self.verbose: + print(j, file=sys.stderr) + r.raise_for_status() + return Machine(datacenter=self, data=j)