From f281406f3273a49ad9c072fd913fc5a3c9d0b1e0 Mon Sep 17 00:00:00 2001 From: Reto Gantenbein Date: Sat, 2 May 2020 17:17:24 +0200 Subject: [PATCH 1/6] Support direct upload/download to Image I/O daemon on oVirt node --- plugins/modules/ovirt_disk.py | 44 ++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index 831a2a1c..5c0042b7 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -186,6 +186,11 @@ - I(True) if the disk should be activated. - When creating disk of virtual machine it is set to I(True). type: bool + use_proxy: + description: + - "Use Image I/O proxy when uploading or downloading disk image. Set + this to I(True) if you cannot directly connect to the oVirt node." + type: bool extends_documentation_fragment: ovirt.ovirt.ovirt ''' @@ -366,7 +371,10 @@ def transfer(connection, module, direction, transfer_func): time.sleep(module.params['poll_interval']) transfer = transfer_service.get() - proxy_url = urlparse(transfer.proxy_url) + if module.params['use_proxy']: + destination_url = urlparse(transfer.proxy_url) + else: + destination_url = urlparse(transfer.transfer_url) context = ssl.create_default_context() auth = module.params['auth'] if auth.get('insecure'): @@ -375,16 +383,16 @@ def transfer(connection, module, direction, transfer_func): elif auth.get('ca_file'): context.load_verify_locations(cafile=auth.get('ca_file')) - proxy_connection = HTTPSConnection( - proxy_url.hostname, - proxy_url.port, + transfer_connection = HTTPSConnection( + destination_url.hostname, + destination_url.port, context=context, ) transfer_func( transfer_service, - proxy_connection, - proxy_url, + transfer_connection, + destination_url, transfer.signed_ticket ) return True @@ -416,17 +424,10 @@ def transfer(connection, module, direction, transfer_func): def download_disk_image(connection, module): - def _transfer(transfer_service, proxy_connection, proxy_url, transfer_ticket): + def _transfer(transfer_service, transfer_connection, destination_url, transfer_ticket): BUF_SIZE = 128 * 1024 - transfer_headers = { - 'Authorization': transfer_ticket, - } - proxy_connection.request( - 'GET', - proxy_url.path, - headers=transfer_headers, - ) - r = proxy_connection.getresponse() + transfer_connection.request('GET', destination_url.path) + r = transfer_connection.getresponse() path = module.params["download_image_path"] image_size = int(r.getheader('Content-Length')) with open(path, "wb") as mydisk: @@ -448,14 +449,14 @@ def _transfer(transfer_service, proxy_connection, proxy_url, transfer_ticket): def upload_disk_image(connection, module): - def _transfer(transfer_service, proxy_connection, proxy_url, transfer_ticket): + def _transfer(transfer_service, transfer_connection, destination_url, transfer_ticket): BUF_SIZE = 128 * 1024 path = module.params['upload_image_path'] image_size = os.path.getsize(path) - proxy_connection.putrequest("PUT", proxy_url.path) - proxy_connection.putheader('Content-Length', "%d" % (image_size,)) - proxy_connection.endheaders() + transfer_connection.putrequest("PUT", destination_url.path) + transfer_connection.putheader('Content-Length', "%d" % (image_size,)) + transfer_connection.endheaders() with open(path, "rb") as disk: pos = 0 while pos < image_size: @@ -464,7 +465,7 @@ def _transfer(transfer_service, proxy_connection, proxy_url, transfer_ticket): if not chunk: transfer_service.pause() raise RuntimeError("Unexpected end of file at pos=%d" % pos) - proxy_connection.send(chunk) + transfer_connection.send(chunk) pos += len(chunk) return transfer( @@ -671,6 +672,7 @@ def main(): host=dict(default=None), wipe_after_delete=dict(type='bool', default=None), activate=dict(default=None, type='bool'), + use_proxy=dict(default=False, type='bool'), ) module = AnsibleModule( argument_spec=argument_spec, From af575d4a6f45c8d844e1589a477ed0e79a49f4a0 Mon Sep 17 00:00:00 2001 From: Reto Gantenbein Date: Sat, 2 May 2020 17:49:18 +0200 Subject: [PATCH 2/6] ovirt_disk: auto-detect proxy connection based on engine version --- plugins/modules/ovirt_disk.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index 5c0042b7..8d1783bb 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -337,6 +337,7 @@ get_dict_of_struct, search_by_name, wait, + engine_supported, ) @@ -371,7 +372,7 @@ def transfer(connection, module, direction, transfer_func): time.sleep(module.params['poll_interval']) transfer = transfer_service.get() - if module.params['use_proxy']: + if module.params['use_proxy'] or not engine_supported(connection, '4.4'): destination_url = urlparse(transfer.proxy_url) else: destination_url = urlparse(transfer.transfer_url) @@ -672,7 +673,7 @@ def main(): host=dict(default=None), wipe_after_delete=dict(type='bool', default=None), activate=dict(default=None, type='bool'), - use_proxy=dict(default=False, type='bool'), + use_proxy=dict(default=None, type='bool'), ) module = AnsibleModule( argument_spec=argument_spec, From 36446c7362fda2c6616162d9eb7a1de54eb524e8 Mon Sep 17 00:00:00 2001 From: mnecas Date: Thu, 21 May 2020 09:38:33 +0200 Subject: [PATCH 3/6] Use proxy as fallback --- plugins/modules/ovirt_disk.py | 50 ++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index 8d1783bb..b104110a 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -186,11 +186,6 @@ - I(True) if the disk should be activated. - When creating disk of virtual machine it is set to I(True). type: bool - use_proxy: - description: - - "Use Image I/O proxy when uploading or downloading disk image. Set - this to I(True) if you cannot directly connect to the oVirt node." - type: bool extends_documentation_fragment: ovirt.ovirt.ovirt ''' @@ -337,7 +332,6 @@ get_dict_of_struct, search_by_name, wait, - engine_supported, ) @@ -372,10 +366,6 @@ def transfer(connection, module, direction, transfer_func): time.sleep(module.params['poll_interval']) transfer = transfer_service.get() - if module.params['use_proxy'] or not engine_supported(connection, '4.4'): - destination_url = urlparse(transfer.proxy_url) - else: - destination_url = urlparse(transfer.transfer_url) context = ssl.create_default_context() auth = module.params['auth'] if auth.get('insecure'): @@ -384,18 +374,35 @@ def transfer(connection, module, direction, transfer_func): elif auth.get('ca_file'): context.load_verify_locations(cafile=auth.get('ca_file')) - transfer_connection = HTTPSConnection( - destination_url.hostname, - destination_url.port, - context=context, - ) + try: + destination_url = urlparse(transfer.transfer_url) + transfer_connection = HTTPSConnection( + destination_url.hostname, + destination_url.port, + context=context, + ) + transfer_func( + transfer_service, + transfer_connection, + destination_url, + transfer.signed_ticket + ) + except Exception as e: + module.warn("Image transfer fail with message: {}.".format(e)) + module.warn("Retrying trasfer with proxy.") + destination_url = urlparse(transfer.proxy_url) + transfer_connection = HTTPSConnection( + destination_url.hostname, + destination_url.port, + context=context, + ) - transfer_func( - transfer_service, - transfer_connection, - destination_url, - transfer.signed_ticket - ) + transfer_func( + transfer_service, + transfer_connection, + destination_url, + transfer.signed_ticket + ) return True finally: transfer_service.finalize() @@ -673,7 +680,6 @@ def main(): host=dict(default=None), wipe_after_delete=dict(type='bool', default=None), activate=dict(default=None, type='bool'), - use_proxy=dict(default=None, type='bool'), ) module = AnsibleModule( argument_spec=argument_spec, From 7c90057b85d937c9ca81eb1f78f59fdba49cb9f4 Mon Sep 17 00:00:00 2001 From: mnecas Date: Thu, 21 May 2020 18:31:29 +0200 Subject: [PATCH 4/6] add create_transfer_connection --- plugins/modules/ovirt_disk.py | 55 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index b104110a..145c507e 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -347,6 +347,25 @@ def _search_by_lun(disks_service, lun_id): return res[0] if res else None +def create_transfer_connection(module, transfer, context, connect_timeout=10, read_timeout=60): + url = urlparse(transfer.transfer_url) + connection = HTTPSConnection( + url.netloc, context=context, timeout=connect_timeout) + try: + connection.connect() + except OSError as e: + # Typically ConnectionRefusedError or socket.gaierror. + module.warn("Cannot connect to %s, trying %s: %s", transfer.transfer_url, transfer.proxy_url, e) + + url = urlparse(transfer.proxy_url) + connection = HTTPSConnection( + url.netloc, context=context, timeout=connect_timeout) + connection.connect() + + connection.sock.settimeout(read_timeout) + return connection, url + + def transfer(connection, module, direction, transfer_func): transfers_service = connection.system_service().image_transfers_service() transfer = transfers_service.add( @@ -374,35 +393,13 @@ def transfer(connection, module, direction, transfer_func): elif auth.get('ca_file'): context.load_verify_locations(cafile=auth.get('ca_file')) - try: - destination_url = urlparse(transfer.transfer_url) - transfer_connection = HTTPSConnection( - destination_url.hostname, - destination_url.port, - context=context, - ) - transfer_func( - transfer_service, - transfer_connection, - destination_url, - transfer.signed_ticket - ) - except Exception as e: - module.warn("Image transfer fail with message: {}.".format(e)) - module.warn("Retrying trasfer with proxy.") - destination_url = urlparse(transfer.proxy_url) - transfer_connection = HTTPSConnection( - destination_url.hostname, - destination_url.port, - context=context, - ) - - transfer_func( - transfer_service, - transfer_connection, - destination_url, - transfer.signed_ticket - ) + transfer_connection, url = create_transfer_connection(module, transfer, context) + transfer_func( + transfer_service, + transfer_connection, + url, + transfer.signed_ticket + ) return True finally: transfer_service.finalize() From c6da921a711724cdffe18cba9d04d79c4de33484 Mon Sep 17 00:00:00 2001 From: mnecas Date: Thu, 21 May 2020 18:36:47 +0200 Subject: [PATCH 5/6] correct warn msg --- plugins/modules/ovirt_disk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index 145c507e..febbd5de 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -355,7 +355,8 @@ def create_transfer_connection(module, transfer, context, connect_timeout=10, re connection.connect() except OSError as e: # Typically ConnectionRefusedError or socket.gaierror. - module.warn("Cannot connect to %s, trying %s: %s", transfer.transfer_url, transfer.proxy_url, e) + module.warn("Cannot connect to {}, trying {}: {}" + .format(transfer.transfer_url, transfer.proxy_url, e)) url = urlparse(transfer.proxy_url) connection = HTTPSConnection( From 0fdab533b2f0e8060b57b8358e85dd4974694b27 Mon Sep 17 00:00:00 2001 From: mnecas Date: Thu, 21 May 2020 20:30:33 +0200 Subject: [PATCH 6/6] update transfer names --- plugins/modules/ovirt_disk.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/modules/ovirt_disk.py b/plugins/modules/ovirt_disk.py index febbd5de..28fd4adb 100644 --- a/plugins/modules/ovirt_disk.py +++ b/plugins/modules/ovirt_disk.py @@ -355,8 +355,7 @@ def create_transfer_connection(module, transfer, context, connect_timeout=10, re connection.connect() except OSError as e: # Typically ConnectionRefusedError or socket.gaierror. - module.warn("Cannot connect to {}, trying {}: {}" - .format(transfer.transfer_url, transfer.proxy_url, e)) + module.warn("Cannot connect to %s, trying %s: %s" % (transfer.transfer_url, transfer.proxy_url, e)) url = urlparse(transfer.proxy_url) connection = HTTPSConnection( @@ -394,12 +393,11 @@ def transfer(connection, module, direction, transfer_func): elif auth.get('ca_file'): context.load_verify_locations(cafile=auth.get('ca_file')) - transfer_connection, url = create_transfer_connection(module, transfer, context) + transfer_connection, transfer_url = create_transfer_connection(module, transfer, context) transfer_func( transfer_service, transfer_connection, - url, - transfer.signed_ticket + transfer_url, ) return True finally: @@ -430,9 +428,9 @@ def transfer(connection, module, direction, transfer_func): def download_disk_image(connection, module): - def _transfer(transfer_service, transfer_connection, destination_url, transfer_ticket): + def _transfer(transfer_service, transfer_connection, transfer_url): BUF_SIZE = 128 * 1024 - transfer_connection.request('GET', destination_url.path) + transfer_connection.request('GET', transfer_url.path) r = transfer_connection.getresponse() path = module.params["download_image_path"] image_size = int(r.getheader('Content-Length')) @@ -455,12 +453,12 @@ def _transfer(transfer_service, transfer_connection, destination_url, transfer_t def upload_disk_image(connection, module): - def _transfer(transfer_service, transfer_connection, destination_url, transfer_ticket): + def _transfer(transfer_service, transfer_connection, transfer_url): BUF_SIZE = 128 * 1024 path = module.params['upload_image_path'] image_size = os.path.getsize(path) - transfer_connection.putrequest("PUT", destination_url.path) + transfer_connection.putrequest("PUT", transfer_url.path) transfer_connection.putheader('Content-Length', "%d" % (image_size,)) transfer_connection.endheaders() with open(path, "rb") as disk: