From 5c0992ce7f65c67b83bb929f4f4ee2d3bf4288a1 Mon Sep 17 00:00:00 2001 From: David Margery Date: Wed, 9 Dec 2015 20:35:03 +0100 Subject: [PATCH 1/6] Make error message when call to AM failed more explicit --- src/gcf/geni/am/am_method_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gcf/geni/am/am_method_context.py b/src/gcf/geni/am/am_method_context.py index c6e11f4c..7b205df1 100644 --- a/src/gcf/geni/am/am_method_context.py +++ b/src/gcf/geni/am/am_method_context.py @@ -157,7 +157,7 @@ def __exit__(self, type, value, traceback_object): self._logger.exception("AM API Error in %s" % self._method_name) self._result=self._api_error(value); elif type: - self._logger.error("Generic Error in %s" % self._method_name) + self._logger.error("Generic Error of type %s in %s" % (str(type),self._method_name)) self._handleError(value) self._logger.info("Result from %s: %s", self._method_name, From 4b84aabebf34193f192ec74ab5819d1ce3c652ed Mon Sep 17 00:00:00 2001 From: David Margery Date: Thu, 11 Oct 2018 11:06:46 +0200 Subject: [PATCH 2/6] Reduce the amount of info INFO debug level prints For large outputs of ListResources, even when debug is not activated, the message built upon exit of each call is far too large, especialy when it is compressed : this brings little value to the log --- src/gcf/geni/am/am_method_context.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/gcf/geni/am/am_method_context.py b/src/gcf/geni/am/am_method_context.py index 7b205df1..53ed427a 100644 --- a/src/gcf/geni/am/am_method_context.py +++ b/src/gcf/geni/am/am_method_context.py @@ -24,6 +24,7 @@ from __future__ import absolute_import import os +import logging import traceback from ...sfa.trust.gid import GID @@ -160,8 +161,13 @@ def __exit__(self, type, value, traceback_object): self._logger.error("Generic Error of type %s in %s" % (str(type),self._method_name)) self._handleError(value) - self._logger.info("Result from %s: %s", self._method_name, - self._result) + if self._logger.isEnabledFor(logging.DEBUG): + self._logger.info("Result from %s: %s", self._method_name, + self._result) + else: + self._logger.info("Result from %s: %s... (turn on debug to get all the output)", + self._method_name, + str(self._result)[0:800]) # Return a GENI_style error return for given exception/traceback def _errorReturn(self, e): From c545da3aa0855eb1f8d08fba617dc3f6d156cdf5 Mon Sep 17 00:00:00 2001 From: David Margery Date: Tue, 21 May 2019 15:55:06 +0200 Subject: [PATCH 3/6] Delegate TLS handling to a frontend A per python docs, the built-in HTTP request handler is not fit for production. Add an option to use geni-tools in production by having it listen on localhost, using http, to be called by a frontend that will handle the TLS handshake and pass the client certificate as an HTTP Header. To use this variant, you need to set proto and certheader options in the configuration file. --- src/gcf-am.py | 26 ++++++++- src/gcf/geni/XMLRPCServer.py | 100 +++++++++++++++++++++++++++++++++++ src/gcf/geni/am/am3.py | 27 ++++++---- 3 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/gcf/geni/XMLRPCServer.py diff --git a/src/gcf-am.py b/src/gcf-am.py index 0c647212..79cee158 100755 --- a/src/gcf-am.py +++ b/src/gcf-am.py @@ -153,13 +153,34 @@ def main(argv=None): resource_manager = \ getInstanceFromClassname(opts.authorizer_resource_manager) + # by default, the agregate manager will expect to be reached through https protocol + # and handle the tls client authentication itself but the python classes used to do + # this are not suitable for production use (as per python's doc). + # Therefore allow for the server to sit behind an apache (or other) frontend doing the + # tls client authentification and only see requests that are forwarded to it, with the + # authenticated client certificate sent to it as an http header + proto = 'https' + if hasattr(opts, 'proto') and opts.proto is not None and str(opts.proto).strip() != "": + if str(opts.proto).strip().lower() == "http": + proto = 'http' + elif str(opts.proto).strip().lower() == "https": + proto = 'https' + else: + proto = 'https' + logging.getLogger('gcf-am').warning("Invalid argument for 'proto', set default : https") + + certheader='X-Geni-Client-Cert' + if proto is 'http': + if hasattr(opts, 'certheader') and opts.certheader is not None and str(opts.certheader).strip() != "": + certheader=str(opts.certheader).strip() + delegate=None if hasattr(opts, 'delegate') and opts.delegate is not None and str(opts.delegate).strip() != "": try: delegate = getInstanceFromClassname(opts.delegate, getAbsPath(opts.rootcadir), config['global']['base_name'], - "https://%s:%d/" % (opts.host, int(opts.port)), + "%s://%s:%d/" % (proto, opts.host, int(opts.port)), **vars(opts) ) except AttributeError, e: @@ -210,7 +231,8 @@ def main(argv=None): base_name=config['global']['base_name'], authorizer=authorizer, resource_manager=resource_manager, - delegate=delegate, multithread=multithread) + delegate=delegate, multithread=multithread, + proto=proto, certheader=certheader) else: msg = "Unknown API version: %d. Valid choices are \"1\", \"2\", or \"3\"" sys.exit(msg % (opts.api_version)) diff --git a/src/gcf/geni/XMLRPCServer.py b/src/gcf/geni/XMLRPCServer.py new file mode 100644 index 00000000..b1f1e4b3 --- /dev/null +++ b/src/gcf/geni/XMLRPCServer.py @@ -0,0 +1,100 @@ +#---------------------------------------------------------------------- +# Copyright (c) 2010-2016 Raytheon BBN Technologies +# Copyright (c) 2019 Inria by David Margery for the Fed4FIRE+ project +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- + +"""A simple XML RPC server supporting getting client cert from HTTP header. + +Based on this article: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81549 + +""" + +from __future__ import absolute_import + +import ssl +import base64 +import textwrap +import os + +from SimpleXMLRPCServer import SimpleXMLRPCServer +from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler +from ..sfa.trust.certificate import Certificate +from OpenSSL import crypto + +class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + """A request handler that grabs the peer's certificate from + the http headers and makes it available while the request is handled + + This implementation can only be used in a single-threaded, one + request at a time model. The peer certificate is stashed on the + XML RPC server at the start of a call and removed at the end of a + call. This is the only way I could find to access this + information. + """ + + def parse_request(self): + SimpleXMLRPCRequestHandler.parse_request(self) + client_cert_string=self.headers.get(self.server.certheader_name, "") + # going through headers in python loose end of line caraters + # massage the string back to proper PEM format + client_cert_string=client_cert_string.replace(' ',"\n") + client_cert_string=client_cert_string.replace("BEGIN\n","BEGIN ") + client_cert_string=client_cert_string.replace("END\n","END ") + + if client_cert_string is "": + self.server.pem_cert = None + self.send_error(400, "Bad request - client cert required") + else: + self.server.pem_cert = client_cert_string + + if self.server.logRequests: + client_cert=Certificate(string=client_cert_string) + self.log_message("Got call from client cert: %s", client_cert.get_printable_subject()) + return True + + + +class XMLRPCServer(SimpleXMLRPCServer): + """An extension to SimpleXMLRPCServer that expects TLS transaction. + has been proxied through a production quality web server, and that the + client's peer cert is passed to it as an http header +""" + + def __init__(self, addr, requestHandler=XMLRPCRequestHandler, + logRequests=False, allow_none=False, encoding=None, + bind_and_activate=True, certheader='X-Geni-Client-Cert'): + SimpleXMLRPCServer.__init__(self, addr, requestHandler, logRequests, + allow_none, encoding, False) + self.certheader_name=certheader + if bind_and_activate: + # This next throws a socket.error on error, eg + # Address already in use or Permission denied. + # Catch for clearer error message? + self.server_bind() + self.server_activate() + + # Return the PEM cert for current XMLRPC client connection + # This works for the single threaded case. Need to override + # This method for the threaded case + def get_pem_cert(self): + return self.pem_cert diff --git a/src/gcf/geni/am/am3.py b/src/gcf/geni/am/am3.py index 0cd727b4..213ef782 100644 --- a/src/gcf/geni/am/am3.py +++ b/src/gcf/geni/am/am3.py @@ -53,6 +53,7 @@ from ..util.tz_util import tzd from ..util.urn_util import publicid_to_urn from ..util import urn_util as urn +from ..XMLRPCServer import XMLRPCServer from ..SecureXMLRPCServer import SecureXMLRPCServer from ..SecureThreadedXMLRPCServer import SecureThreadedXMLRPCServer @@ -1387,29 +1388,37 @@ def __init__(self, addr, keyfile=None, certfile=None, trust_roots_dir=None, ca_certs=None, base_name=None, authorizer=None, resource_manager=None, - delegate=None, multithread=False): - # ca_certs arg here must be a file of concatenated certs - if ca_certs is None: - raise Exception('Missing CA Certs') - elif not os.path.isfile(os.path.expanduser(ca_certs)): - raise Exception('CA Certs must be an existing file of accepted root certs: %s' % ca_certs) + delegate=None, multithread=False, + proto='https', certheader=None): + use_https=proto=='https' + if use_https: + # ca_certs arg here must be a file of concatenated certs + if ca_certs is None: + raise Exception('Missing CA Certs') + elif not os.path.isfile(os.path.expanduser(ca_certs)): + raise Exception('CA Certs must be an existing file of accepted root certs: %s' % ca_certs) # Decode the addr into a URL. Is there a pythonic way to do this? - server_url = "https://%s:%d/" % addr + server_url = "%s://%s:%d/" % (proto, addr[0], addr[1]) if delegate is None: delegate = ReferenceAggregateManager(trust_roots_dir, base_name, server_url) # FIXED: set logRequests=true if --debug logRequest=logging.getLogger().getEffectiveLevel()==logging.DEBUG - if multithread: + if multithread and use_https: self._server = SecureThreadedXMLRPCServer(addr, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, logRequests=logRequest) - else: + elif not multithread and use_https: self._server = SecureXMLRPCServer(addr, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, logRequests=logRequest) + else: + self._server = XMLRPCServer(addr, + logRequests=logRequest, + certheader=certheader) + aggregate_manager = AggregateManager(trust_roots_dir, delegate, authorizer, resource_manager) self._server.register_instance(aggregate_manager) From 12f5f70f6a6ed5e0112cb62125c3c4209fb82814 Mon Sep 17 00:00:00 2001 From: David Margery Date: Tue, 21 May 2019 16:01:19 +0200 Subject: [PATCH 4/6] Ensure client cert is given when using http proto --- src/gcf/geni/XMLRPCServer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gcf/geni/XMLRPCServer.py b/src/gcf/geni/XMLRPCServer.py index b1f1e4b3..6e96f684 100644 --- a/src/gcf/geni/XMLRPCServer.py +++ b/src/gcf/geni/XMLRPCServer.py @@ -53,6 +53,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler): """ def parse_request(self): + parse_success=True SimpleXMLRPCRequestHandler.parse_request(self) client_cert_string=self.headers.get(self.server.certheader_name, "") # going through headers in python loose end of line caraters @@ -63,6 +64,7 @@ def parse_request(self): if client_cert_string is "": self.server.pem_cert = None + parse_success=False self.send_error(400, "Bad request - client cert required") else: self.server.pem_cert = client_cert_string @@ -70,7 +72,7 @@ def parse_request(self): if self.server.logRequests: client_cert=Certificate(string=client_cert_string) self.log_message("Got call from client cert: %s", client_cert.get_printable_subject()) - return True + return parse_success From 53f46f3c76e324f87c124cceacfef40b010d238f Mon Sep 17 00:00:00 2001 From: David Margery Date: Tue, 21 May 2019 15:55:06 +0200 Subject: [PATCH 5/6] Delegate TLS handling to a frontend A per python docs, the built-in HTTP request handler is not fit for production. Add an option to use geni-tools in production by having it listen on localhost, using http, to be called by a frontend that will handle the TLS handshake and pass the client certificate as an HTTP Header. To use this variant, you need to set proto and certheader options in the configuration file. --- src/gcf-am.py | 26 ++++++++- src/gcf/geni/XMLRPCServer.py | 100 +++++++++++++++++++++++++++++++++++ src/gcf/geni/am/am3.py | 27 ++++++---- 3 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/gcf/geni/XMLRPCServer.py diff --git a/src/gcf-am.py b/src/gcf-am.py index 0c647212..79cee158 100755 --- a/src/gcf-am.py +++ b/src/gcf-am.py @@ -153,13 +153,34 @@ def main(argv=None): resource_manager = \ getInstanceFromClassname(opts.authorizer_resource_manager) + # by default, the agregate manager will expect to be reached through https protocol + # and handle the tls client authentication itself but the python classes used to do + # this are not suitable for production use (as per python's doc). + # Therefore allow for the server to sit behind an apache (or other) frontend doing the + # tls client authentification and only see requests that are forwarded to it, with the + # authenticated client certificate sent to it as an http header + proto = 'https' + if hasattr(opts, 'proto') and opts.proto is not None and str(opts.proto).strip() != "": + if str(opts.proto).strip().lower() == "http": + proto = 'http' + elif str(opts.proto).strip().lower() == "https": + proto = 'https' + else: + proto = 'https' + logging.getLogger('gcf-am').warning("Invalid argument for 'proto', set default : https") + + certheader='X-Geni-Client-Cert' + if proto is 'http': + if hasattr(opts, 'certheader') and opts.certheader is not None and str(opts.certheader).strip() != "": + certheader=str(opts.certheader).strip() + delegate=None if hasattr(opts, 'delegate') and opts.delegate is not None and str(opts.delegate).strip() != "": try: delegate = getInstanceFromClassname(opts.delegate, getAbsPath(opts.rootcadir), config['global']['base_name'], - "https://%s:%d/" % (opts.host, int(opts.port)), + "%s://%s:%d/" % (proto, opts.host, int(opts.port)), **vars(opts) ) except AttributeError, e: @@ -210,7 +231,8 @@ def main(argv=None): base_name=config['global']['base_name'], authorizer=authorizer, resource_manager=resource_manager, - delegate=delegate, multithread=multithread) + delegate=delegate, multithread=multithread, + proto=proto, certheader=certheader) else: msg = "Unknown API version: %d. Valid choices are \"1\", \"2\", or \"3\"" sys.exit(msg % (opts.api_version)) diff --git a/src/gcf/geni/XMLRPCServer.py b/src/gcf/geni/XMLRPCServer.py new file mode 100644 index 00000000..b1f1e4b3 --- /dev/null +++ b/src/gcf/geni/XMLRPCServer.py @@ -0,0 +1,100 @@ +#---------------------------------------------------------------------- +# Copyright (c) 2010-2016 Raytheon BBN Technologies +# Copyright (c) 2019 Inria by David Margery for the Fed4FIRE+ project +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and/or hardware specification (the "Work") to +# deal in the Work without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Work, and to permit persons to whom the Work +# is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Work. +# +# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS +# IN THE WORK. +#---------------------------------------------------------------------- + +"""A simple XML RPC server supporting getting client cert from HTTP header. + +Based on this article: + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/81549 + +""" + +from __future__ import absolute_import + +import ssl +import base64 +import textwrap +import os + +from SimpleXMLRPCServer import SimpleXMLRPCServer +from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler +from ..sfa.trust.certificate import Certificate +from OpenSSL import crypto + +class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + """A request handler that grabs the peer's certificate from + the http headers and makes it available while the request is handled + + This implementation can only be used in a single-threaded, one + request at a time model. The peer certificate is stashed on the + XML RPC server at the start of a call and removed at the end of a + call. This is the only way I could find to access this + information. + """ + + def parse_request(self): + SimpleXMLRPCRequestHandler.parse_request(self) + client_cert_string=self.headers.get(self.server.certheader_name, "") + # going through headers in python loose end of line caraters + # massage the string back to proper PEM format + client_cert_string=client_cert_string.replace(' ',"\n") + client_cert_string=client_cert_string.replace("BEGIN\n","BEGIN ") + client_cert_string=client_cert_string.replace("END\n","END ") + + if client_cert_string is "": + self.server.pem_cert = None + self.send_error(400, "Bad request - client cert required") + else: + self.server.pem_cert = client_cert_string + + if self.server.logRequests: + client_cert=Certificate(string=client_cert_string) + self.log_message("Got call from client cert: %s", client_cert.get_printable_subject()) + return True + + + +class XMLRPCServer(SimpleXMLRPCServer): + """An extension to SimpleXMLRPCServer that expects TLS transaction. + has been proxied through a production quality web server, and that the + client's peer cert is passed to it as an http header +""" + + def __init__(self, addr, requestHandler=XMLRPCRequestHandler, + logRequests=False, allow_none=False, encoding=None, + bind_and_activate=True, certheader='X-Geni-Client-Cert'): + SimpleXMLRPCServer.__init__(self, addr, requestHandler, logRequests, + allow_none, encoding, False) + self.certheader_name=certheader + if bind_and_activate: + # This next throws a socket.error on error, eg + # Address already in use or Permission denied. + # Catch for clearer error message? + self.server_bind() + self.server_activate() + + # Return the PEM cert for current XMLRPC client connection + # This works for the single threaded case. Need to override + # This method for the threaded case + def get_pem_cert(self): + return self.pem_cert diff --git a/src/gcf/geni/am/am3.py b/src/gcf/geni/am/am3.py index 0cd727b4..213ef782 100644 --- a/src/gcf/geni/am/am3.py +++ b/src/gcf/geni/am/am3.py @@ -53,6 +53,7 @@ from ..util.tz_util import tzd from ..util.urn_util import publicid_to_urn from ..util import urn_util as urn +from ..XMLRPCServer import XMLRPCServer from ..SecureXMLRPCServer import SecureXMLRPCServer from ..SecureThreadedXMLRPCServer import SecureThreadedXMLRPCServer @@ -1387,29 +1388,37 @@ def __init__(self, addr, keyfile=None, certfile=None, trust_roots_dir=None, ca_certs=None, base_name=None, authorizer=None, resource_manager=None, - delegate=None, multithread=False): - # ca_certs arg here must be a file of concatenated certs - if ca_certs is None: - raise Exception('Missing CA Certs') - elif not os.path.isfile(os.path.expanduser(ca_certs)): - raise Exception('CA Certs must be an existing file of accepted root certs: %s' % ca_certs) + delegate=None, multithread=False, + proto='https', certheader=None): + use_https=proto=='https' + if use_https: + # ca_certs arg here must be a file of concatenated certs + if ca_certs is None: + raise Exception('Missing CA Certs') + elif not os.path.isfile(os.path.expanduser(ca_certs)): + raise Exception('CA Certs must be an existing file of accepted root certs: %s' % ca_certs) # Decode the addr into a URL. Is there a pythonic way to do this? - server_url = "https://%s:%d/" % addr + server_url = "%s://%s:%d/" % (proto, addr[0], addr[1]) if delegate is None: delegate = ReferenceAggregateManager(trust_roots_dir, base_name, server_url) # FIXED: set logRequests=true if --debug logRequest=logging.getLogger().getEffectiveLevel()==logging.DEBUG - if multithread: + if multithread and use_https: self._server = SecureThreadedXMLRPCServer(addr, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, logRequests=logRequest) - else: + elif not multithread and use_https: self._server = SecureXMLRPCServer(addr, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, logRequests=logRequest) + else: + self._server = XMLRPCServer(addr, + logRequests=logRequest, + certheader=certheader) + aggregate_manager = AggregateManager(trust_roots_dir, delegate, authorizer, resource_manager) self._server.register_instance(aggregate_manager) From 4ed21dcd37fe4b5d3e0627dafedc109271447915 Mon Sep 17 00:00:00 2001 From: David Margery Date: Tue, 21 May 2019 16:01:19 +0200 Subject: [PATCH 6/6] Ensure client cert is given when using http proto --- src/gcf/geni/XMLRPCServer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gcf/geni/XMLRPCServer.py b/src/gcf/geni/XMLRPCServer.py index b1f1e4b3..6e96f684 100644 --- a/src/gcf/geni/XMLRPCServer.py +++ b/src/gcf/geni/XMLRPCServer.py @@ -53,6 +53,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCRequestHandler): """ def parse_request(self): + parse_success=True SimpleXMLRPCRequestHandler.parse_request(self) client_cert_string=self.headers.get(self.server.certheader_name, "") # going through headers in python loose end of line caraters @@ -63,6 +64,7 @@ def parse_request(self): if client_cert_string is "": self.server.pem_cert = None + parse_success=False self.send_error(400, "Bad request - client cert required") else: self.server.pem_cert = client_cert_string @@ -70,7 +72,7 @@ def parse_request(self): if self.server.logRequests: client_cert=Certificate(string=client_cert_string) self.log_message("Got call from client cert: %s", client_cert.get_printable_subject()) - return True + return parse_success