diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log index 832ed5062c7..45e2d4c2cf8 100755 --- a/bin/cylc-cat-log +++ b/bin/cylc-cat-log @@ -42,7 +42,7 @@ import traceback from cylc.option_parsers import CylcOptionParser as COP from cylc.rundb import CylcSuiteDAO -from cylc.suite_host import is_remote +from cylc.hostuserutil import is_remote from cylc.suite_logging import get_logs from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.task_id import TaskID diff --git a/bin/cylc-documentation b/bin/cylc-documentation index b1f019e323b..ad8907833e7 100755 --- a/bin/cylc-documentation +++ b/bin/cylc-documentation @@ -47,7 +47,6 @@ from optparse import OptionParser import cylc.flags from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.run_get_stdout import run_get_stdout -from cylc.suite_host import get_hostname, get_user def main(): @@ -73,13 +72,13 @@ def main(): "--user", help="Other user account name. This results in " "command reinvocation on the remote account.", - metavar="USER", default=get_user(), action="store", dest="owner") + metavar="USER", action="store", dest="owner") parser.add_option( "--host", help="Other host name. This results in " "command reinvocation on the remote account.", - metavar="HOST", action="store", default=get_hostname(), dest="host") + metavar="HOST", action="store", dest="host") parser.add_option( "--debug", diff --git a/bin/cylc-gscan b/bin/cylc-gscan index 6e0aafd4b3c..de213e2e0bc 100755 --- a/bin/cylc-gscan +++ b/bin/cylc-gscan @@ -43,7 +43,7 @@ gtk.settings_get_default().set_long_property( from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.gui.gscan import ScanApp from cylc.option_parsers import CylcOptionParser as COP -from cylc.suite_host import get_user +from cylc.hostuserutil import get_user def main(): diff --git a/bin/cylc-scan b/bin/cylc-scan index a09d743a7e7..b5b95be540a 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -44,13 +44,13 @@ if "--use-ssh" in sys.argv[1:]: import re +from cylc.cfgspec.globalcfg import GLOBAL_CFG +from cylc.hostuserutil import get_user from cylc.network.port_scan import scan_many from cylc.option_parsers import CylcOptionParser as COP -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.suite_host import get_user from cylc.suite_status import ( KEY_DESCRIPTION, KEY_META, KEY_NAME, KEY_OWNER, KEY_STATES, - KEY_TASKS_BY_STATE, KEY_TITLE, KEY_UPDATE_TIME) + KEY_TITLE, KEY_UPDATE_TIME) from cylc.task_state import TASK_STATUSES_ORDERED from cylc.task_state_prop import get_status_prop diff --git a/doc/src/cylc-user-guide/siterc.tex b/doc/src/cylc-user-guide/siterc.tex index fda7e99c121..92c6886c258 100644 --- a/doc/src/cylc-user-guide/siterc.tex +++ b/doc/src/cylc-user-guide/siterc.tex @@ -769,7 +769,7 @@ \subsection{[suite host self-identification] } other methods. For the {\em address} method, cylc attempts to use a special external ``target address'' to determine the IP address of the suite host as seen by remote task hosts (in-source documentation in -\lstinline=$CYLC_DIR/lib/cylc/suite_host.py= explains how this works). +\lstinline=$CYLC_DIR/lib/cylc/hostuserutil.py= explains how this works). And finally, as a last resort, you can choose the {\em hardwired} method and manually specify the host name or IP address of the suite host. diff --git a/lib/cylc/cfgspec/globalcfg.py b/lib/cylc/cfgspec/globalcfg.py index bb3b88ff22d..c3a8a454f20 100644 --- a/lib/cylc/cfgspec/globalcfg.py +++ b/lib/cylc/cfgspec/globalcfg.py @@ -28,7 +28,7 @@ from parsec.validate import coercers from parsec import ParsecError from parsec.upgrade import upgrader, converter -from cylc.suite_host import is_remote_user +from cylc.hostuserutil import is_remote_user from cylc.envvar import expandvars from cylc.mkdir_p import mkdir_p import cylc.flags diff --git a/lib/cylc/gui/app_gcylc.py b/lib/cylc/gui/app_gcylc.py index bd83c1fd4ab..16db52ec4bf 100644 --- a/lib/cylc/gui/app_gcylc.py +++ b/lib/cylc/gui/app_gcylc.py @@ -29,7 +29,7 @@ from uuid import uuid4 from isodatetime.parsers import TimePointParser -from cylc.suite_host import is_remote, is_remote_host, is_remote_user +from cylc.hostuserutil import is_remote, is_remote_host, is_remote_user from cylc.gui.dbchooser import dbchooser from cylc.gui.combo_logviewer import ComboLogViewer from cylc.gui.warning_dialog import warning_dialog, info_dialog diff --git a/lib/cylc/gui/dbchooser.py b/lib/cylc/gui/dbchooser.py index f87b5219931..b3aa80495b7 100644 --- a/lib/cylc/gui/dbchooser.py +++ b/lib/cylc/gui/dbchooser.py @@ -28,7 +28,7 @@ from cylc.network.port_scan import scan_many from cylc.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.run_get_stdout import run_get_stdout -from cylc.suite_host import is_remote_host, is_remote_user +from cylc.hostuserutil import is_remote_host, is_remote_user class db_updater(threading.Thread): diff --git a/lib/cylc/gui/gpanel.py b/lib/cylc/gui/gpanel.py index be4346c0725..223bff9211b 100755 --- a/lib/cylc/gui/gpanel.py +++ b/lib/cylc/gui/gpanel.py @@ -35,7 +35,7 @@ from cylc.gui.scanutil import (KEY_PORT, get_gpanel_scan_menu, update_suites_info) from cylc.gui.util import get_icon, setup_icons -from cylc.suite_host import get_user +from cylc.hostuserutil import get_user from cylc.suite_status import KEY_STATES from cylc.task_state_prop import extract_group_state diff --git a/lib/cylc/host_select.py b/lib/cylc/host_select.py index 79ca533bd1f..4be7b330632 100644 --- a/lib/cylc/host_select.py +++ b/lib/cylc/host_select.py @@ -22,7 +22,7 @@ import re from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.run_get_stdout import run_get_stdout -from cylc.suite_host import is_remote_host +from cylc.hostuserutil import is_remote_host REC_COMMAND = re.compile(r"(`|\$\()\s*(.*)\s*(`|\))$") diff --git a/lib/cylc/suite_host.py b/lib/cylc/hostuserutil.py similarity index 83% rename from lib/cylc/suite_host.py rename to lib/cylc/hostuserutil.py index 0b4e62f74fa..b9c4f400472 100644 --- a/lib/cylc/suite_host.py +++ b/lib/cylc/hostuserutil.py @@ -77,9 +77,8 @@ def get_inst(cls, new=False, expire=None): def __init__(self, expire): self.expire_time = time() + expire - self._host_name = None - self._host_ip_address = None - self._host_name_pref = None + self._host = None # preferred name of localhost + self._host_exs = {} # host: socket.gethostbyname_ex(host), ... self._remote_hosts = {} # host: is_remote, ... self.user_pwent = None self.remote_users = {} @@ -112,18 +111,13 @@ def get_host_ip_by_name(target): """Return internal IP address of target.""" return socket.gethostbyname(target) - def get_hostname(self): - """Return the fully qualified domain name for current host.""" - if self._host_name is None: - self._host_name = socket.getfqdn() - return self._host_name - - def _get_host_ip_address(self): - """Return the external IP address for the current host.""" - if self._host_ip_address is None: - self._host_ip_address = self.get_local_ip_address( - self._get_identification_cfg('target')) - return self._host_ip_address + def _get_host_info(self, target=None): + """Return the extended info of the current host.""" + if target not in self._host_exs: + if target is None: + target = socket.getfqdn() + self._host_exs[target] = socket.gethostbyname_ex(target) + return self._host_exs[target] @staticmethod def _get_identification_cfg(key): @@ -131,7 +125,7 @@ def _get_identification_cfg(key): from cylc.cfgspec.globalcfg import GLOBAL_CFG return GLOBAL_CFG.get(['suite host self-identification', key]) - def get_suite_host(self): + def get_host(self): """Return the preferred identifier for the suite (or current) host. As specified by the "suite host self-identification" settings in the @@ -139,16 +133,21 @@ def get_suite_host(self): identification by task jobs. """ - if self._host_name_pref is None: + if self._host is None: hardwired = self._get_identification_cfg('host') method = self._get_identification_cfg('method') if method == 'address': - self._host_name_pref = self._get_host_ip_address() + self._host = self.get_local_ip_address( + self._get_identification_cfg('target')) elif method == 'hardwired' and hardwired: - self._host_name_pref = hardwired + self._host = hardwired else: # if method == 'name': - self._host_name_pref = self.get_hostname() - return self._host_name_pref + self._host = self._get_host_info()[0] + return self._host + + def get_fqdn_by_host(self, target): + """Return the fully qualified domain name of the target host.""" + return self._get_host_info(target)[0] def get_user(self): """Return name of current user.""" @@ -178,15 +177,12 @@ def is_remote_host(self, name): self._remote_hosts[name] = False else: try: - ipa = socket.gethostbyname(name) + host_info = self._get_host_info(name) except IOError: self._remote_hosts[name] = True else: - host_ip_address = self._get_host_ip_address() - # local IP address of the suite host (e.g. 127.0.0.1) - my_ip_address = socket.gethostbyname(self.get_hostname()) self._remote_hosts[name] = ( - ipa not in [host_ip_address, my_ip_address]) + host_info != self._get_host_info()) return self._remote_hosts[name] def is_remote_user(self, name): @@ -210,9 +206,9 @@ def is_remote(self, host, owner): return self.is_remote_host(host) or self.is_remote_user(owner) -def get_hostname(): - """Shorthand for HostUtil.get_inst().get_hostname().""" - return HostUtil.get_inst().get_hostname() +def get_host_ip_by_name(target): + """Shorthand for HostUtil.get_inst().get_host_ip_by_name(target).""" + return HostUtil.get_inst().get_host_ip_by_name(target) def get_local_ip_address(target): @@ -220,14 +216,14 @@ def get_local_ip_address(target): return HostUtil.get_inst().get_local_ip_address(target) -def get_host_ip_by_name(target): - """Shorthand for HostUtil.get_inst().get_host_ip_by_name(target).""" - return HostUtil.get_inst().get_host_ip_by_name(target) +def get_host(): + """Shorthand for HostUtil.get_inst().get_host().""" + return HostUtil.get_inst().get_host() -def get_suite_host(): - """Shorthand for HostUtil.get_inst().get_suite_host().""" - return HostUtil.get_inst().get_suite_host() +def get_fqdn_by_host(target): + """Shorthand for HostUtil.get_inst().get_fqdn_by_host(target).""" + return HostUtil.get_inst().get_fqdn_by_host(target) def get_user(): @@ -266,7 +262,6 @@ def test_hosts(self): self.assertFalse(is_remote_host(None)) self.assertFalse(is_remote_host('localhost')) self.assertFalse(is_remote_host(os.getenv('HOSTNAME'))) - self.assertFalse(is_remote_host(get_hostname())) - self.assertFalse(is_remote_host(get_suite_host())) + self.assertFalse(is_remote_host(get_host())) unittest.main() diff --git a/lib/cylc/network/httpclient.py b/lib/cylc/network/httpclient.py index c97241f77f7..9314b0f9602 100644 --- a/lib/cylc/network/httpclient.py +++ b/lib/cylc/network/httpclient.py @@ -28,7 +28,7 @@ from cylc.exceptions import CylcError import cylc.flags from cylc.network import NO_PASSPHRASE -from cylc.suite_host import get_suite_host, get_user +from cylc.hostuserutil import get_host, get_fqdn_by_host, get_user from cylc.suite_srv_files_mgr import ( SuiteSrvFilesManager, SuiteServiceFileError) from cylc.unicode_util import utf8_enforce @@ -250,19 +250,19 @@ def _call_server_func(self, *func_dicts, **fargs): self._load_contact_info() except (IOError, ValueError, SuiteServiceFileError): raise ClientInfoError(self.suite) - host = self.host - if host == 'localhost': - host = get_suite_host() - + if self.host and self.host.split('.')[0] == 'localhost': + self.host = get_host() + elif self.host and '.' not in self.host: # Not IP and no domain + self.host = get_fqdn_by_host(self.host) http_request_items = [] try: # dictionary containing: url, payload, method http_request_items.append(self._compile_url( - func_dict, host, self.comms_protocol)) + func_dict, self.host, self.comms_protocol)) except (IndexError, ValueError, AttributeError): for f_dict in func_dicts: http_request_items.append(self._compile_url( - f_dict, host, self.comms_protocol)) + f_dict, self.host, self.comms_protocol)) # Remove proxy settings from environment for now environ = {} for key in ("http_proxy", "https_proxy"): @@ -480,7 +480,7 @@ def _get_headers(self): CYLC_VERSION, self.prog_name, self.my_uuid ) ) - auth_info = "%s@%s" % (get_user(), get_suite_host()) + auth_info = "%s@%s" % (get_user(), get_host()) return {"User-Agent": user_agent_string, "From": auth_info} @@ -494,7 +494,7 @@ def _load_contact_info(self): return if self.port: # In case the contact file is corrupted, user can specify the port. - self.host = get_suite_host() + self.host = get_host() return data = self.srv_files_mgr.load_contact_file( self.suite, self.owner, self.host) diff --git a/lib/cylc/network/httpserver.py b/lib/cylc/network/httpserver.py index 7ba72d279c1..4ac77564f91 100644 --- a/lib/cylc/network/httpserver.py +++ b/lib/cylc/network/httpserver.py @@ -36,7 +36,7 @@ from cylc.network import ( NO_PASSPHRASE, PRIVILEGE_LEVELS, PRIV_IDENTITY, PRIV_DESCRIPTION, PRIV_STATE_TOTALS, PRIV_FULL_READ, PRIV_SHUTDOWN, PRIV_FULL_CONTROL) -from cylc.suite_host import get_suite_host +from cylc.hostuserutil import get_host from cylc.suite_logging import ERR, LOG from cylc.suite_srv_files_mgr import ( SuiteSrvFilesManager, SuiteServiceFileError) @@ -121,7 +121,7 @@ def start(self): """Start quick web service.""" # cherrypy.config["tools.encode.on"] = True # cherrypy.config["tools.encode.encoding"] = "utf-8" - cherrypy.config["server.socket_host"] = get_suite_host() + cherrypy.config["server.socket_host"] = get_host() cherrypy.config["engine.autoreload.on"] = False if self.comms_method == "https": diff --git a/lib/cylc/network/port_scan.py b/lib/cylc/network/port_scan.py index 9eab0f6a231..59a9d3ed98f 100644 --- a/lib/cylc/network/port_scan.py +++ b/lib/cylc/network/port_scan.py @@ -29,7 +29,7 @@ SuiteRuntimeServiceClient, ClientError, ClientTimeout) from cylc.suite_srv_files_mgr import ( SuiteSrvFilesManager, SuiteServiceFileError) -from cylc.suite_host import is_remote_host, get_host_ip_by_name +from cylc.hostuserutil import is_remote_host, get_host_ip_by_name CONNECT_TIMEOUT = 5.0 INACTIVITY_TIMEOUT = 10.0 diff --git a/lib/cylc/option_parsers.py b/lib/cylc/option_parsers.py index 4430502b0e6..45c793d0e40 100644 --- a/lib/cylc/option_parsers.py +++ b/lib/cylc/option_parsers.py @@ -20,7 +20,6 @@ from optparse import OptionParser, OptionConflictError import os import cylc.flags -from cylc.suite_host import get_user class CylcOptionParser(OptionParser): @@ -115,8 +114,7 @@ def add_std_options(self): "Other user account name. This results in " "command reinvocation on the remote account." ), - metavar="USER", default=get_user(), - action="store", dest="owner") + metavar="USER", action="store", dest="owner") self.add_std_option( "--host", help="Other host name. This results in " diff --git a/lib/cylc/remote.py b/lib/cylc/remote.py index 23c24f14dc7..8c9304060e6 100644 --- a/lib/cylc/remote.py +++ b/lib/cylc/remote.py @@ -67,7 +67,7 @@ def __init__(self, argv=None): if self.owner is None and self.host is None: self.is_remote = False else: - from cylc.suite_host import is_remote + from cylc.hostuserutil import is_remote self.is_remote = is_remote(self.host, self.owner) def execute(self, force_required=False, env=None, path=None, diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index 49745528c93..dec2208f3d1 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -46,7 +46,7 @@ from cylc.suite_db_mgr import SuiteDatabaseManager from cylc.suite_events import ( SuiteEventContext, SuiteEventError, SuiteEventHandler) -from cylc.suite_host import get_suite_host, get_user +from cylc.hostuserutil import get_host, get_user from cylc.suite_logging import SuiteLog, ERR, LOG from cylc.suite_srv_files_mgr import ( SuiteSrvFilesManager, SuiteServiceFileError) @@ -149,7 +149,7 @@ def __init__(self, is_restart, options, args): self.run_mode = self.options.run_mode self.owner = get_user() - self.host = get_suite_host() + self.host = get_host() self.port = None self.is_stalled = False diff --git a/lib/cylc/suite_events.py b/lib/cylc/suite_events.py index cce5a8669c7..9d1efd2bf9b 100644 --- a/lib/cylc/suite_events.py +++ b/lib/cylc/suite_events.py @@ -23,7 +23,7 @@ from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.mp_pool import SuiteProcContext -from cylc.suite_host import get_suite_host, get_user +from cylc.hostuserutil import get_host, get_user from cylc.suite_logging import LOG @@ -106,7 +106,7 @@ def _run_event_mail(self, config, ctx): '-s', subject, '-r', self.get_events_conf( config, - 'mail from', 'notifications@' + get_suite_host()), + 'mail from', 'notifications@' + get_host()), self.get_events_conf(config, 'mail to', get_user()), ], env=env, diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index ff5f8fe0cec..95abd06fb5f 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -26,8 +26,8 @@ import cylc.flags from cylc.mkdir_p import mkdir_p -from cylc.suite_host import ( - get_local_ip_address, get_suite_host, get_user, is_remote, is_remote_host) +from cylc.hostuserutil import ( + get_local_ip_address, get_host, get_user, is_remote, is_remote_host) class SuiteServiceFileError(Exception): @@ -80,7 +80,7 @@ def cache_passphrase(self, reg, owner, host, value): if owner is None: owner = get_user() if host is None: - host = get_suite_host() + host = get_host() path = self._get_cache_dir(reg, owner, host) self.cache[self.FILE_BASE_PASSPHRASE][(reg, owner, host)] = value # Dump to a file only for remote suites loaded via SSH. @@ -237,7 +237,7 @@ def get_auth_item(self, item, reg, owner=None, host=None, content=False): if my_owner is None: my_owner = get_user() if my_host is None: - my_host = get_suite_host() + my_host = get_host() try: return self.cache[item][(reg, my_owner, my_host)] except KeyError: @@ -461,7 +461,7 @@ def _get_ssl_cert(self, path, pkey_obj): # OpenSSL not installed, so we can't use HTTPS anyway. return # Use suite host as the 'common name', but no more than 64 chars. - host = get_suite_host() + host = get_host() common_name = host if len(common_name) > 64: common_name = common_name[:61] + "..." @@ -569,7 +569,7 @@ def _is_local_auth_ok(self, reg, owner, host): if owner is None: owner = get_user() if host is None: - host = get_suite_host() + host = get_host() host_value = data.get(self.KEY_HOST, "") self.can_use_load_auths[(reg, owner, host)] = ( reg == data.get(self.KEY_NAME) and diff --git a/lib/cylc/task_events_mgr.py b/lib/cylc/task_events_mgr.py index 7b293659498..ea5af01a2a9 100644 --- a/lib/cylc/task_events_mgr.py +++ b/lib/cylc/task_events_mgr.py @@ -43,7 +43,7 @@ import cylc.flags from cylc.mp_pool import SuiteProcContext from cylc.suite_logging import ERR, LOG -from cylc.suite_host import get_suite_host, get_user +from cylc.hostuserutil import get_host, get_user from cylc.task_action_timer import TaskActionTimer from cylc.task_message import TaskMessage from cylc.task_state import ( @@ -799,7 +799,7 @@ def _setup_event_mail(self, itask, event): self._get_events_conf( # mail_from itask, "mail from", - "notifications@" + get_suite_host(), + "notifications@" + get_host(), ), self._get_events_conf(itask, "mail to", get_user()), # mail_to self._get_events_conf(itask, "mail smtp"), # mail_smtp diff --git a/lib/cylc/task_job_mgr.py b/lib/cylc/task_job_mgr.py index b750d730abe..6848747ab02 100644 --- a/lib/cylc/task_job_mgr.py +++ b/lib/cylc/task_job_mgr.py @@ -47,7 +47,7 @@ from cylc.job_file import JobFileWriter from cylc.mkdir_p import mkdir_p from cylc.mp_pool import SuiteProcPool, SuiteProcContext -from cylc.suite_host import is_remote, is_remote_host, is_remote_user +from cylc.hostuserutil import is_remote, is_remote_host, is_remote_user from cylc.suite_logging import LOG from cylc.task_events_mgr import TaskEventsManager from cylc.task_message import TaskMessage @@ -364,7 +364,7 @@ def unlink_hosts_contacts(self, reg): del procs[user_at_host] out, err = proc.communicate() if proc.wait(): - ERR.warning(RemoteJobHostInitError( + LOG.warning(RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, ' '.join(quote(item) for item in cmd), proc.returncode, out, err)) @@ -376,7 +376,7 @@ def unlink_hosts_contacts(self, reg): pass out, err = proc.communicate() if proc.wait(): - ERR.warning(RemoteJobHostInitError( + LOG.warning(RemoteJobHostInitError( RemoteJobHostInitError.MSG_TIDY, user_at_host, ' '.join(quote(item) for item in cmd), proc.returncode, out, err)) diff --git a/tests/suite-host-self-id/00-address.t b/tests/suite-host-self-id/00-address.t index 27f5a8ab059..6e4d7afcd83 100644 --- a/tests/suite-host-self-id/00-address.t +++ b/tests/suite-host-self-id/00-address.t @@ -24,7 +24,7 @@ install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" get_local_ip_address() { python - "$1" <<'__PYTHON__' import sys -from cylc.suite_host import get_local_ip_address +from cylc.hostuserutil import get_local_ip_address sys.stdout.write("%s\n" % get_local_ip_address(sys.argv[1])) __PYTHON__ } diff --git a/tests/suite-host-self-id/01-unit.t b/tests/suite-host-self-id/01-unit.t index e13fd1d5cbb..5fee6769de5 100755 --- a/tests/suite-host-self-id/01-unit.t +++ b/tests/suite-host-self-id/01-unit.t @@ -19,5 +19,5 @@ . "$(dirname "$0")/test_header" set_test_number 1 -run_ok "${TEST_NAME_BASE}" python -m 'cylc.suite_host' +run_ok "${TEST_NAME_BASE}" python -m 'cylc.hostuserutil' exit