From 982cf726debf598e0ce574228aba91213b64fd2a Mon Sep 17 00:00:00 2001 From: "devender.mishra" Date: Thu, 24 Nov 2016 16:07:19 +0530 Subject: [PATCH 1/2] Add method for direct download --- glanceclient/common/http.py | 50 +++++++++++++++++++++++++++++++++++++ glanceclient/v2/images.py | 23 +++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index ee53f7da9..9e18f05db 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -288,3 +288,53 @@ def _close_after_stream(response, chunk_size): # This will return the connection to the HTTPConnectionPool in urllib3 # and ideally reduce the number of HTTPConnectionPool full warnings. response.close() + +def direct_download_from_link(direct_link, chunk_size=1024): + session = requests.Session() + session.headers["User-Agent"] = USER_AGENT + try: + resp = session.request('GET', direct_link, stream=True) + except requests.exceptions.Timeout as e: + message = ("Error communicating with %(url)s: %(e)s" % + dict(url=direct_link, e=e)) + raise exc.InvalidEndpoint(message=message) + except (requests.exceptions.ConnectionError, ProtocolError) as e: + message = ("Error finding address for %(url)s: %(e)s" % + dict(url=direct_link, e=e)) + raise exc.CommunicationError(message=message) + except socket.gaierror as e: + message = "Error finding address for %s: %s" % ( + direct_link, e) + raise exc.InvalidEndpoint(message=message) + except (socket.error, socket.timeout) as e: + endpoint = self.endpoint + message = ("Error communicating with %(endpoint)s %(e)s" % + {'endpoint': direct_link, 'e': e}) + raise exc.CommunicationError(message=message) + + resp, body_iter = _direct_download_handle_response(resp, chunk_size) + HTTPClient.log_http_response(resp) + return resp, body_iter + +def _direct_download_handle_response(resp, chunk_size=1024): + if not resp.ok: + LOG.debug("Request returned failure status %s." % resp.status_code) + raise exc.from_response(resp, resp.content) + + content_type = resp.headers.get('Content-Type') + + # Read body into string if it isn't obviously image data + if content_type == 'application/octet-stream': + # Do not read all response in memory when downloading an image. + body_iter = _close_after_stream(resp, CHUNKSIZE) + else: + content = resp.text + if content_type and content_type.startswith('application/json'): + # Let's use requests json method, it should take care of + # response encoding + body_iter = resp.json() + else: + body_iter = resp.iter_content(chunk_size) + + return resp, body_iter + diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index f7ae92712..c1835d4c4 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -21,6 +21,7 @@ import warlock from glanceclient.common import utils +from glanceclient.common import http from glanceclient import exc from glanceclient.v2 import schemas @@ -195,6 +196,28 @@ def data(self, image_id, do_checksum=True): return utils.IterableWithLength(body, content_length) + def data_direct_download(self, image_id, do_checksum=True, chunk_size=1024): + """Retrieve data of an image without overloading glance server (using direct download link of the image) + + :param image_id: ID of the image to download. + :param do_checksum: Enable/disable checksum validation. + Similar to data but get the link to image and download on its own. + """ + url = 'v2/images/%s/link' % image_id + resp, body = self.http_client.get(url) + direct_link = body.get('image_path', None) + checksum = body.get('image_checksum', None) + + if not direct_link: + raise Exception("Image '%s' does not have a valid direct download path" % image_id) + + resp, body = http.direct_download_from_link(direct_link, chunk_size) + content_length = int(resp.headers.get('content-length', 0)) + + if do_checksum and checksum is not None: + body = utils.integrity_iter(body, checksum) + return utils.IterableWithLength(body, content_length) + def upload(self, image_id, image_data, image_size=None): """ Upload the data for an image. From 6ebdf0c78ed0862a8eb4f36e52ab7b8e0fd40b98 Mon Sep 17 00:00:00 2001 From: Devender Mishra Date: Tue, 29 Nov 2016 16:11:11 +0530 Subject: [PATCH 2/2] Fix stream issue and add get_link call --- glanceclient/common/http.py | 20 +++----------------- glanceclient/v2/images.py | 7 ++++++- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index 9e18f05db..53f847e8e 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -289,7 +289,7 @@ def _close_after_stream(response, chunk_size): # and ideally reduce the number of HTTPConnectionPool full warnings. response.close() -def direct_download_from_link(direct_link, chunk_size=1024): +def direct_download_from_link(direct_link, chunk_size=CHUNKSIZE): session = requests.Session() session.headers["User-Agent"] = USER_AGENT try: @@ -316,25 +316,11 @@ def direct_download_from_link(direct_link, chunk_size=1024): HTTPClient.log_http_response(resp) return resp, body_iter -def _direct_download_handle_response(resp, chunk_size=1024): +def _direct_download_handle_response(resp, chunk_size=CHUNKSIZE): if not resp.ok: LOG.debug("Request returned failure status %s." % resp.status_code) raise exc.from_response(resp, resp.content) - - content_type = resp.headers.get('Content-Type') - # Read body into string if it isn't obviously image data - if content_type == 'application/octet-stream': - # Do not read all response in memory when downloading an image. - body_iter = _close_after_stream(resp, CHUNKSIZE) - else: - content = resp.text - if content_type and content_type.startswith('application/json'): - # Let's use requests json method, it should take care of - # response encoding - body_iter = resp.json() - else: - body_iter = resp.iter_content(chunk_size) - + body_iter = _close_after_stream(resp, chunk_size) return resp, body_iter diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index c1835d4c4..46473d9a9 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -196,7 +196,12 @@ def data(self, image_id, do_checksum=True): return utils.IterableWithLength(body, content_length) - def data_direct_download(self, image_id, do_checksum=True, chunk_size=1024): + def get_link(self, image_id): + url = '/v2/images/%s/link' % image_id + resp, body = self.http_client.get(url) + return self.model(**body) + + def data_direct_download(self, image_id, do_checksum=True, chunk_size=64*1024): """Retrieve data of an image without overloading glance server (using direct download link of the image) :param image_id: ID of the image to download.