From 427e35ee48809f6de60abc849073d93976a4f6be Mon Sep 17 00:00:00 2001 From: David Kwon Date: Tue, 28 Jun 2022 11:08:43 -0400 Subject: [PATCH] fix: add extra steps for monitoring docs (#2379) * fix: add extra steps for monitoring docs Signed-off-by: David Kwon Co-authored-by: Max Leonov --- .../snip_che-create-monitoring-namespace.adoc | 4 ++ .../pages/configuring-observability.adoc | 2 +- .../installing-prometheus-and-grafana.adoc | 21 ++++++- .../upgrading-che-7-41-on-openshift.adoc | 2 +- ...ollecting-che-metrics-with-prometheus.adoc | 42 +++++++++---- ...pace-operator-metrics-with-prometheus.adoc | 63 +++++++++---------- ...ing-che-metrics-on-grafana-dashboards.adoc | 2 +- ...perator-metrics-on-grafana-dashboards.adoc | 2 +- 8 files changed, 86 insertions(+), 52 deletions(-) create mode 100644 modules/administration-guide/examples/snip_che-create-monitoring-namespace.adoc diff --git a/modules/administration-guide/examples/snip_che-create-monitoring-namespace.adoc b/modules/administration-guide/examples/snip_che-create-monitoring-namespace.adoc new file mode 100644 index 00000000000..a8bd887bd96 --- /dev/null +++ b/modules/administration-guide/examples/snip_che-create-monitoring-namespace.adoc @@ -0,0 +1,4 @@ +[source,terminal,subs="+attributes,quotes"] +---- +$ {orch-cli} create {orch-namespace} monitoring +---- diff --git a/modules/administration-guide/pages/configuring-observability.adoc b/modules/administration-guide/pages/configuring-observability.adoc index 3b00daa8223..f3b2aeab975 100644 --- a/modules/administration-guide/pages/configuring-observability.adoc +++ b/modules/administration-guide/pages/configuring-observability.adoc @@ -12,5 +12,5 @@ To configure {prod-short} observability features, see: * xref:che-theia-workspaces.adoc[] * xref:configuring-server-logging.adoc[] * xref:collecting-logs-using-chectl.adoc[] -* xref:monitoring-che.adoc[] * xref:monitoring-the-dev-workspace-operator.adoc[] +* xref:monitoring-che.adoc[] diff --git a/modules/administration-guide/pages/installing-prometheus-and-grafana.adoc b/modules/administration-guide/pages/installing-prometheus-and-grafana.adoc index 05ebd571901..866fcadd049 100644 --- a/modules/administration-guide/pages/installing-prometheus-and-grafana.adoc +++ b/modules/administration-guide/pages/installing-prometheus-and-grafana.adoc @@ -9,7 +9,7 @@ [id="installing-prometheus-and-grafana"] = Installing Prometheus and Grafana -You can install Prometheus and Grafana by applying `template.yaml` that consists of a Deployment and Service for both Prometheus and Grafana. +You can install Prometheus and Grafana by applying `template.yaml`. The `template.yaml` file in this example provides a monitoring stack of basic configuration, Deployments and Services to get started with Prometheus and Grafana. Alternatively, you can use the link:https://github.com/prometheus-operator/prometheus-operator[Prometheus Operator] and link:https://github.com/grafana-operator/grafana-operator[Grafana Operator]. @@ -21,7 +21,16 @@ Alternatively, you can use the link:https://github.com/prometheus-operator/prome To install Prometheus and Grafana by using `template.yaml`: -* Apply `template.yaml` to the cluster by running `{orch-cli} apply -f template.yaml`. +. Create a new {orch-namespace}, `monitoring`, for Prometheus and Grafana: ++ +include::example$snip_{project-context}-create-monitoring-namespace.adoc[] + +. Apply `template.yaml` in the `monitoring` {orch-namespace}: ++ +[subs="+attributes,quotes"] +---- +$ {orch-cli} apply -f template.yaml -n monitoring +---- .`template.yaml` ==== @@ -95,6 +104,7 @@ spec: labels: app: prometheus spec: + serviceAccountName: prometheus containers: - image: quay.io/prometheus/prometheus:v2.36.0 name: prometheus @@ -120,7 +130,12 @@ kind: ConfigMap metadata: name: prometheus-config data: - prometheus.yml: | + prometheus.yml: "" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus --- ---- ==== diff --git a/modules/administration-guide/pages/upgrading-che-7-41-on-openshift.adoc b/modules/administration-guide/pages/upgrading-che-7-41-on-openshift.adoc index 85569ee0c6c..a94661c532d 100644 --- a/modules/administration-guide/pages/upgrading-che-7-41-on-openshift.adoc +++ b/modules/administration-guide/pages/upgrading-che-7-41-on-openshift.adoc @@ -7,7 +7,7 @@ [id="upgrading-{prod-prev-id-short}-{prod-last-version-pre-dwo}-on-openshift_{context}"] = Upgrading {prod-prev-short} {prod-last-version-pre-dwo} on OpenShift -The workspace engine and authentication system used up to and including {prod-prev-short} {prod-last-version-pre-dwo} are deprecated. Due to that, upgrading {prod-prev-short} {prod-last-version-pre-dwo} to {prod-short} {prod-ver} involves a manual procedure. +The workspace engine and authentication system used up to and including {prod-prev-short} {prod-last-version-pre-dwo} are deprecated. Due to that, upgrading {prod-prev-short} {prod-last-version-pre-dwo} to {prod-short} {prod-ver} involves running upgrade scripts. include::partial$proc_manually-upgrading-che-7-41-on-openshift.adoc[leveloffset=+1] include::partial$proc_rolling-the-upgrade-back-to-che-7-41-on-openshift.adoc[leveloffset=+1] diff --git a/modules/administration-guide/partials/proc_collecting-che-metrics-with-prometheus.adoc b/modules/administration-guide/partials/proc_collecting-che-metrics-with-prometheus.adoc index f2ffffba9a3..5f1fd72138e 100644 --- a/modules/administration-guide/partials/proc_collecting-che-metrics-with-prometheus.adoc +++ b/modules/administration-guide/partials/proc_collecting-che-metrics-with-prometheus.adoc @@ -9,15 +9,17 @@ To use Prometheus to collect, store, and query JVM metrics for {prod-short} Serv * {prod-short} is exposing metrics on port `8087`. See xref:enabling-and-exposing-{prod-id-short}-metrics_{context}[Enabling and exposing {prod-short} server JVM metrics]. -* Prometheus 2.26.0 or later is running. The Prometheus console is running on port `9090` with a corresponding `service` and `route`. See link:https://prometheus.io/docs/introduction/first_steps/[First steps with Prometheus]. +* Prometheus 2.26.0 or later is running. The Prometheus console is running on port `9090` with a corresponding Service. See link:https://prometheus.io/docs/introduction/first_steps/[First steps with Prometheus]. .Procedure -* Configure Prometheus to scrape metrics from port `8087`. +. Configure Prometheus to scrape metrics from port `8087`. ++ +NOTE: The xref:installing-prometheus-and-grafana.adoc[example monitoring stack] already creates the `prometheus-config` ConfigMap with an empty configuration. To provide the Prometheus configuration details, edit the `data` field of the ConfigMap. + .Prometheus configuration ==== -[source,yaml,subs="+attributes"] +[source,yaml,subs="+quotes,+attributes,+macros"] ---- apiVersion: v1 kind: ConfigMap @@ -28,24 +30,38 @@ data: global: scrape_interval: 5s <1> evaluation_interval: 5s <2> - scrape_configs: <3> - - job_name: 'che' + scrape_configs: <3> + - job_name: 'Che Server' static_configs: - - targets: ['[che-host]:8087'] <4> + - targets: ['che-host.__<{prod-short}_{orch-namespace}>__:8087'] <4> ---- <1> The rate at which a target is scraped. <2> The rate at which the recording and alerting rules are re-checked. -<3> The Resources that Prometheus monitors. In the default configuration, a single job, `che`, scrapes the time series data exposed by {prod-short} Server. -<4> The scrape metrics from port `8087`. +<3> The resources that Prometheus monitors. In the default configuration, a single job, `Che Server`, scrapes the time series data exposed by {prod-short} Server. +<4> The scrape target for the metrics from port `8087`. Replace `__<{prod-short}_{orch-namespace}>__` with the {prod-short} {orch-namespace}. The default {prod-short} {orch-namespace} is `{prod-namespace}`. ==== -.Verification steps - -. View the metrics in the Prometheus console at `http://____/metrics`. +. Scale the `Prometheus` Deployment down and up to read the updated ConfigMap from the previous step. ++ +[source,terminal,subs="+attributes,quotes"] +---- +$ {orch-cli} scale --replicas=0 deployment/prometheus -n monitoring && {orch-cli} scale --replicas=1 deployment/prometheus -n monitoring +---- -. Query the metrics in the Prometheus console from `http://____/graph`. For more information, see link:https://prometheus.io/docs/introduction/first_steps/#using-the-expression-browser[Using the expression browser]. +.Verification -. Verify that all targets are up by viewing the targets endpoint at `http://____/targets`. +. Use port forwarding to access the `Prometheus` Service locally: ++ +[source,terminal,subs="+attributes,quotes"] +---- +$ {orch-cli} port-forward svc/prometheus 9090:9090 -n monitoring +---- +. Verify that all targets are up by viewing the `targets` endpoint at `localhost:9090/targets`. +. Use the Prometheus console to view and query metrics: +** View metrics at `localhost:9090/metrics`. +** Query metrics from `localhost:9090/graph`. ++ +For more information, see link:https://prometheus.io/docs/introduction/first_steps/#using-the-expression-browser[Using the expression browser]. .Additional resources diff --git a/modules/administration-guide/partials/proc_collecting-dev-workspace-operator-metrics-with-prometheus.adoc b/modules/administration-guide/partials/proc_collecting-dev-workspace-operator-metrics-with-prometheus.adoc index ceb699a61bc..7f01a97848e 100644 --- a/modules/administration-guide/partials/proc_collecting-dev-workspace-operator-metrics-with-prometheus.adoc +++ b/modules/administration-guide/partials/proc_collecting-dev-workspace-operator-metrics-with-prometheus.adoc @@ -6,33 +6,17 @@ To use Prometheus to collect, store, and query metrics about the {devworkspace} .Prerequisites -* The link:https://github.com/devfile/devworkspace-operator/blob/v0.10.0/deploy/deployment/kubernetes/objects/devworkspace-controller-metrics.Service.yaml[`devworkspace-controller-metrics` Service] is exposing metrics on port `8443`. +* The `devworkspace-controller-metrics` Service is exposing metrics on port `8443`. This is preconfigured by default. -* The `devworkspace-webhookserver` Service is exposing metrics on port `9443`. By default, the Service exposes metrics on port `9443`. +* The `devworkspace-webhookserver` Service is exposing metrics on port `9443`. This is preconfigured by default. -* Prometheus 2.26.0 or later is running. The Prometheus console is running on port `9090` with a corresponding `service` and `route`. See link:https://prometheus.io/docs/introduction/first_steps/[First steps with Prometheus]. +* Prometheus 2.26.0 or later is running. The Prometheus console is running on port `9090` with a corresponding Service. See link:https://prometheus.io/docs/introduction/first_steps/[First steps with Prometheus]. .Procedure -. Create a `ClusterRoleBinding` to bind the `ServiceAccount` associated with Prometheus to the link:https://github.com/devfile/devworkspace-operator/blob/main/deploy/deployment/kubernetes/objects/devworkspace-controller-metrics-reader.ClusterRole.yaml[devworkspace-controller-metrics-reader] `ClusterRole`. +. Create a ClusterRoleBinding to bind the ServiceAccount associated with Prometheus to the link:https://github.com/devfile/devworkspace-operator/blob/main/deploy/deployment/kubernetes/objects/devworkspace-controller-metrics-reader.ClusterRole.yaml[devworkspace-controller-metrics-reader] ClusterRole. For the xref:installing-prometheus-and-grafana.adoc[example monitoring stack], the name of the ServiceAccount to be be used is `prometheus`. + -NOTE: Without the `ClusterRoleBinding`, you cannot access {devworkspace} metrics because access is protected with role-based access control (RBAC). -+ -.ClusterRole -==== -[source,yaml,subs="+quotes,+attributes,+macros"] ----- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: devworkspace-controller-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ----- -==== +NOTE: Without the ClusterRoleBinding, you cannot access {devworkspace} metrics because access is protected with role-based access control (RBAC). + .ClusterRoleBinding ==== @@ -44,8 +28,8 @@ metadata: name: devworkspace-controller-metrics-binding subjects: - kind: ServiceAccount - name: ____ - namespace: ____ + name: prometheus + namespace: monitoring roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -55,6 +39,8 @@ roleRef: . Configure Prometheus to scrape metrics from port `8443` exposed by the `devworkspace-controller-metrics` Service and from port `9443` exposed by the `devworkspace-webhookserver` Service. + +NOTE: The xref:installing-prometheus-and-grafana.adoc[example monitoring stack] already creates the `prometheus-config` ConfigMap with an empty configuration. To provide the Prometheus configuration details, edit the `data` field of the ConfigMap. ++ .Prometheus configuration ==== [source,yaml,subs="+quotes,+attributes,+macros"] @@ -63,6 +49,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: prometheus-config + namespace: monitoring data: prometheus.yml: |- global: @@ -77,7 +64,7 @@ data: tls_config: insecure_skip_verify: true static_configs: - - targets: ['devworkspace-controller-metrics:8443'] <4> + - targets: ['devworkspace-controller-metrics.____:8443'] <4> - job_name: 'DevWorkspace webhooks' scheme: https authorization: @@ -86,24 +73,36 @@ data: tls_config: insecure_skip_verify: true static_configs: - - targets: ['devworkspace-webhookserver:9443'] <5> + - targets: ['devworkspace-webhookserver.____:9443'] <5> ---- <1> The rate at which a target is scraped. <2> The rate at which the recording and alerting rules are re-checked. <3> The resources that Prometheus monitors. In the default configuration, two jobs, `DevWorkspace` and `DevWorkspace webhooks`, scrape the time series data exposed by the `devworkspace-controller-metrics` and `devworkspace-webhookserver` Services. -<4> The scrape metrics from port `8443`. -<5> The scrape metrics from port `9443`. +<4> The scrape target for the metrics from port `8443`. Replace `____` with the {orch-namespace} where the `devworkspace-controller-metrics` `Service` is located. +<5> The scrape target for the metrics from port `9443`. Replace `____` with the {orch-namespace} where the `devworkspace-webhookserver` `Service` is located. ==== -.Verification steps +. Scale the `Prometheus` Deployment down and up to read the updated ConfigMap from the previous step. ++ +[source,terminal,subs="+attributes,quotes"] +---- +$ {orch-cli} scale --replicas=0 deployment/prometheus -n monitoring && {orch-cli} scale --replicas=1 deployment/prometheus -n monitoring +---- + +.Verification +. Use port forwarding to access the `Prometheus` Service locally: ++ +[source,terminal,subs="+attributes,quotes"] +---- +$ {orch-cli} port-forward svc/prometheus 9090:9090 -n monitoring +---- +. Verify that all targets are up by viewing the targets endpoint at `localhost:9090/targets`. . Use the Prometheus console to view and query metrics: -** View the metrics at `http://____/metrics`. -** Query metrics from `http://____/graph`. +** View metrics at `localhost:9090/metrics`. +** Query metrics from `localhost:9090/graph`. + For more information, see link:https://prometheus.io/docs/introduction/first_steps/#using-the-expression-browser[Using the expression browser]. -. Verify that all targets are up by viewing the targets endpoint at `http://____/targets`. - [role="_additional-resources"] .Additional resources diff --git a/modules/administration-guide/partials/proc_viewing-che-metrics-on-grafana-dashboards.adoc b/modules/administration-guide/partials/proc_viewing-che-metrics-on-grafana-dashboards.adoc index c0f842ac82e..bfe201bec9e 100644 --- a/modules/administration-guide/partials/proc_viewing-che-metrics-on-grafana-dashboards.adoc +++ b/modules/administration-guide/partials/proc_viewing-che-metrics-on-grafana-dashboards.adoc @@ -9,7 +9,7 @@ To view the {prod-short} Server metrics on Grafana: * Prometheus is collecting metrics on the {prod-short} cluster. See xref:monitoring-with-prometheus-and-grafana.adoc[]. -* Grafana 6.0 or later is running on port `3000` with a corresponding `service` and `route`. See link:https://grafana.com/docs/grafana/latest/installation/kubernetes/[Installing Grafana]. +* Grafana 6.0 or later is running on port `3000` with a corresponding Service. See link:https://grafana.com/docs/grafana/latest/installation/kubernetes/[Installing Grafana]. .Procedure diff --git a/modules/administration-guide/partials/proc_viewing-dev-workspace-operator-metrics-on-grafana-dashboards.adoc b/modules/administration-guide/partials/proc_viewing-dev-workspace-operator-metrics-on-grafana-dashboards.adoc index 1fdf1139480..518be577acf 100644 --- a/modules/administration-guide/partials/proc_viewing-dev-workspace-operator-metrics-on-grafana-dashboards.adoc +++ b/modules/administration-guide/partials/proc_viewing-dev-workspace-operator-metrics-on-grafana-dashboards.adoc @@ -9,7 +9,7 @@ To view the {devworkspace} Operator metrics on Grafana with the example dashboar * Grafana version 7.5.3 or later. -* Grafana is running on port `3000` with a corresponding `service` and `route`. See link:https://grafana.com/docs/grafana/latest/installation/kubernetes/[Installing Grafana]. +* Grafana is running on port `3000` with a corresponding Service. See link:https://grafana.com/docs/grafana/latest/installation/kubernetes/[Installing Grafana].