-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
move fluentd config check to separate file
- Loading branch information
1 parent
a7a2729
commit 75f7a0f
Showing
4 changed files
with
322 additions
and
161 deletions.
There are no files selected for viewing
173 changes: 173 additions & 0 deletions
173
roles/openshift_health_checker/openshift_checks/fluentd_config.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| """ | ||
| Module for performing checks on an Fluentd logging deployment configuration | ||
| """ | ||
|
|
||
| from openshift_checks import OpenShiftCheck, OpenShiftCheckException, get_var | ||
|
|
||
| import json | ||
| import os | ||
|
|
||
|
|
||
| class FluentdConfig(OpenShiftCheck): | ||
| """Module that checks logging configuration of an integrated logging Fluentd deployment""" | ||
| name = "fluentd_config" | ||
| tags = ["health"] | ||
|
|
||
| logging_namespace = None | ||
|
|
||
| @classmethod | ||
| def is_active(cls, task_vars): | ||
| return super(FluentdConfig, cls).is_active(task_vars) | ||
|
|
||
| def run(self, tmp, task_vars): | ||
| """Check that Fluentd has running pods, and that its logging config matches Docker's logging config.""" | ||
|
|
||
| self.logging_namespace = get_var(task_vars, "openshift_logging_namespace", default="logging") | ||
| fluentd_pods, error = self.get_pods_for_component( | ||
| self.execute_module, | ||
| self.logging_namespace, | ||
| "fluentd", | ||
| task_vars, | ||
| ) | ||
| if error: | ||
| return {"failed": True, "changed": False, "msg": error} | ||
|
|
||
| running_fluentd_pods = self.running_pods(fluentd_pods) | ||
| if not len(running_fluentd_pods): | ||
| msg = ('No Fluentd pods were found to be in the "Running" state.' | ||
| 'At least one Fluentd pod is required in order to perform this check.') | ||
|
|
||
| return {"failed": True, "changed": False, "msg": msg} | ||
|
|
||
| config_error = self.check_logging_config(fluentd_pods, task_vars) | ||
| if config_error: | ||
| msg = ("The following Fluentd logging configuration problem was found:" | ||
| "\n-------\n" | ||
| "{}".format(config_error)) | ||
| return {"failed": True, "changed": False, "msg": msg} | ||
|
|
||
| return {"failed": False, "changed": False, "msg": 'No problems found with Fluentd logging configuration.'} | ||
|
|
||
| def check_logging_config(self, fluentd_pods, task_vars): | ||
| """Ensure that the configured Docker logging driver matches fluentd settings. | ||
| This means that, at least for now, if the following condition is met: | ||
| openshift_logging_fluentd_use_journal == True | ||
| then the value of the configured Docker logging driver should be "journald". | ||
| Otherwise, the value of the Docker logging driver should be "json-file". | ||
| Returns an error string if the above condition is not met, or None otherwise.""" | ||
|
|
||
| pod_name = fluentd_pods[0]["metadata"]["name"] | ||
| try: | ||
| use_journald = self.exec_oc( | ||
| self.execute_module, | ||
| self.logging_namespace, | ||
| "exec {} /bin/printenv USE_JOURNAL".format(pod_name), | ||
| [], task_vars | ||
| ) | ||
| except OpenShiftCheckException as error: | ||
| if "false" not in str(error): | ||
| return str(error) | ||
|
|
||
| use_journald = False | ||
|
|
||
| docker_info = self.execute_module("docker_info", {}, task_vars) | ||
| if not docker_info.get("info", False): | ||
| return "No information was returned by the Docker client. Unable to verify the logging driver in use." | ||
|
|
||
| if not docker_info["info"].get("LoggingDriver", False): | ||
| return "No logging driver information was returned by the Docker client." | ||
|
|
||
| logging_driver = docker_info["info"]["LoggingDriver"] | ||
| recommended_logging_driver = "journald" | ||
| error = None | ||
|
|
||
| # If fluentd is set to use journald but Docker is not, recommend setting the `--log-driver` | ||
| # option as an inventory file variable, or adding the log driver value as part of the | ||
| # Docker configuration in /etc/docker/daemon.json. There is no global --log-driver flag that | ||
| # can be passed to the Docker binary; the only other recommendation that can be made, would be | ||
| # to pass the `--log-driver` flag to the "run" sub-command of the `docker` binary when running | ||
| # individual containers. | ||
| if use_journald and logging_driver != "journald": | ||
| error = ('Your Fluentd configuration is set to aggregate Docker container logs from "journald".\n' | ||
| 'This differs from your Docker configuration, which has been set to use "{driver}" ' | ||
| 'as the default method of storing logs.\n' | ||
| 'This discrepancy in configuration will prevent Fluentd from receiving any logs' | ||
| 'from your Docker containers.\n').format(driver=logging_driver) | ||
| elif not use_journald and logging_driver != "json-file": | ||
| recommended_logging_driver = "json-file" | ||
| error = ('Your Fluentd configuration is set to aggregate Docker container logs from ' | ||
| 'individual json log files per container.\n ' | ||
| 'This differs from your Docker configuration, which has been set to use ' | ||
| '"{driver}" as the default method of storing logs.\n' | ||
| 'This discrepancy in configuration will prevent Fluentd from receiving any logs' | ||
| 'from your Docker containers.\n').format(driver=logging_driver) | ||
|
|
||
| if error: | ||
| error += ('To resolve this issue, add the following variable to your Ansible inventory file:\n\n' | ||
| ' openshift_docker_options="--log-driver={driver}"\n\n' | ||
| 'Alternatively, you can add the following option to your Docker configuration, located in' | ||
| '"/etc/docker/daemon.json":\n\n' | ||
| '{{ "log-driver": "{driver}" }}\n\n' | ||
| 'See https://docs.docker.com/engine/admin/logging/json-file ' | ||
| 'for more information.').format(driver=recommended_logging_driver) | ||
|
|
||
| return error | ||
|
|
||
| def get_pods_for_component(self, execute_module, namespace, logging_component, task_vars): | ||
| """Get all pods for a given component. Returns: list of pods for component, error string.""" | ||
| pod_output = self.exec_oc( | ||
| execute_module, | ||
| namespace, | ||
| "get pods -l component={} -o json".format(logging_component), | ||
| [], | ||
| task_vars | ||
| ) | ||
| try: | ||
| pods = json.loads(pod_output) | ||
| if not pods or not pods.get('items'): | ||
| raise ValueError() | ||
| except ValueError: | ||
| # successful run but non-parsing data generally means there were no pods in the namespace | ||
| return None, 'There are no pods in the {} namespace. Is logging deployed?'.format(namespace) | ||
|
|
||
| return pods['items'], None | ||
|
|
||
| @staticmethod | ||
| def running_pods(pods): | ||
| """Returns: list of pods in a running state.""" | ||
| return [ | ||
| pod for pod in pods | ||
| if pod['status']['phase'] == 'Running' | ||
| ] | ||
|
|
||
| @staticmethod | ||
| def exec_oc(execute_module=None, namespace="logging", cmd_str="", extra_args=None, task_vars=None): | ||
| """Execute an 'oc' command in the remote host. | ||
| Returns: output of command and namespace, | ||
| or raises OpenShiftCheckException on error.""" | ||
| config_base = get_var(task_vars, "openshift", "common", "config_base") | ||
| args = { | ||
| "namespace": namespace, | ||
| "config_file": os.path.join(config_base, "master", "admin.kubeconfig"), | ||
| "cmd": cmd_str, | ||
| "extra_args": list(extra_args) if extra_args else [], | ||
| } | ||
|
|
||
| result = execute_module("ocutil", args, task_vars) | ||
| if result.get("failed"): | ||
| msg = ( | ||
| 'Unexpected error using `oc` to validate the logging stack components.\n' | ||
| 'Error executing `oc {cmd}`:\n' | ||
| '{error}' | ||
| ).format(cmd=args['cmd'], error=result['result']) | ||
|
|
||
| if result['result'] == '[Errno 2] No such file or directory': | ||
| msg = ( | ||
| "This host is supposed to be a master but does not have the `oc` command where expected.\n" | ||
| "Has an installation been run on this host yet?" | ||
| ) | ||
| raise OpenShiftCheckException(msg) | ||
|
|
||
| return result.get("result", "") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.