Better error messages and allow dashboard deletion

README.md

@@ -153,6 +153,8 @@ If you provide a `base_folder` only json files in that folder (and any subfolder
 
 If you do not provide a dashboards url a set of default dashboards from [DC/OS grafana-dashboards](https://github.com/dcos/grafana-dashboards) (developed and provided by d2iq) are used. Note that these dashboards are for version 2.0.x of DC/OS. If you have a different version you should provide a url for those (select the branch for your version from [https://github.com/dcos/grafana-dashboards/branches](https://github.com/dcos/grafana-dashboards/branches) and select "Download ZIP"). In that case you also need to change `base_folder` to `grafana-dashboards-<version>/dashboards`.
 
+If you want to remove all existing dashboards/folders before uploading the new ones (to force grafana to have a consistent state without local changes) add either `"clear_dashboards": true` (will delete dashboards but keep folders intact) or `"delete_folders": true` (will delete all folders and all dashboards within them) to the `grafana.dashboards` config object.
+
 ### Grafana admin credentials (only for DC/OS EE)
 
 You can configure custom grafana admin credentials by supplying secrets for them.

files/svc.yml

@@ -497,6 +497,7 @@ pods:
             GRAFANA_URL=http://localhost:3000 \
             GRAFANA_DATASOURCE_URL=http://query.${FRAMEWORK_VIP_HOST}:19192 \
             GRAFANA_CLEAR_DASHBOARDS={{GRAFANA_DASHBOARDS_CLEAR_DASHBOARDS}} \
+            GRAFANA_DELETE_FOLDERS={{GRAFANA_DASHBOARDS_DELETE_FOLDERS}} \
             FETCHER_URL={{{GRAFANA_DASHBOARDS_URL}}} \
             FETCHER_BASE_FOLDER={{GRAFANA_DASHBOARDS_BASE_FOLDER}} \
             ./grafana-loader ${LOADER_COMMAND} &
@@ -549,6 +550,7 @@ pods:
           FETCHER_URL={{{GRAFANA_DASHBOARDS_URL}}} \
           FETCHER_BASE_FOLDER={{GRAFANA_DASHBOARDS_BASE_FOLDER}} \
           GRAFANA_CLEAR_DASHBOARDS={{GRAFANA_DASHBOARDS_CLEAR_DASHBOARDS}} \
+          GRAFANA_DELETE_FOLDERS={{GRAFANA_DASHBOARDS_DELETE_FOLDERS}} \
           GRAFANA_URL=http://localhost:3000 \
           ./grafana-loader dashboards
         cpus: 0.1
@@ -595,14 +597,14 @@ plans:
           - default: [[reload-config]]
   {{/ALERTMANAGER_CONFIG_CONTENT_BASE64}}
   {{/ALERTMANAGER_ENABLED}}
-  {{#PROMETHEUS_RULES_URL}}
+  {{#PROMETHEUS_ALERTING_RULES_URL}}
   reload-prometheus-alert-rules:
     phases:
       reload:
         pod: prometheus
         steps:
           - default: [[reload-rules]]
-  {{/PROMETHEUS_RULES_URL}}
+  {{/PROMETHEUS_ALERTING_RULES_URL}}
   {{#GRAFANA_ENABLED}}
   {{#GRAFANA_DASHBOARDS_ENABLED}}
   reload-grafana-dashboards:

universe/config.json

@@ -313,7 +313,12 @@
               "default": ""
             },
             "clear_dashboards": {
-              "description": "Delete all existing dashboards and folders during loading of new dashboards",
+              "description": "Delete all existing dashboards before loading new dashboards",
+              "type": "boolean",
+              "default": false
+            },
+            "delete_folders": {
+              "description": "Delete all existing folders (and implicitly all dashboards) before loading new dashboards",
               "type": "boolean",
               "default": false
             }

universe/marathon.json.mustache

@@ -84,6 +84,7 @@
     "GRAFANA_DASHBOARDS_BASE_FOLDER": "{{grafana.dashboards.base_folder}}",
     {{/grafana.dashboards.url}}
     "GRAFANA_DASHBOARDS_CLEAR_DASHBOARDS": "{{grafana.dashboards.clear_dashboards}}",
+    "GRAFANA_DASHBOARDS_DELETE_FOLDERS": "{{grafana.dashboards.delete_folders}}",
     "GRAFANA_ADMIN_USERNAME_SECRET": "{{grafana.admin.username_secret}}",
     "GRAFANA_ADMIN_PASSWORD_SECRET": "{{grafana.admin.password_secret}}",
     {{#grafana.config_base64}}

universe/package.json

@@ -1,16 +1,16 @@
 
 {
   "packagingVersion": "4.0",
-  "upgradesFrom": ["0.1.0"],
-  "downgradesTo": ["0.1.0"],
+  "upgradesFrom": ["0.1.0", "0.1.1"],
+  "downgradesTo": ["0.1.0", "0.1.1"],
   "minDcosReleaseVersion": "1.13",
   "name": "thanos",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "maintainer": "MaibornWolff",
   "description": "HA monitoring stack based on thanos",
   "selected": false,
   "framework": true,
   "tags": ["thanos", "prometheus", "grafana", "pushgateway", "alertmanager"],
-  "postInstallNotes": "DC/OS YOURNAMEHERE is being installed!\n\n\tDocumentation: {{documentation-path}}\n\tIssues: {{issues-path}}",
-  "postUninstallNotes": "DC/OS YOURNAMEHERE is being uninstalled."
+  "postInstallNotes": "DC/OS thanos is being installed!\n\n\tDocumentation: https://github.com/MaibornWolff/dcos-thanos\n\tIssues: https://github.com/MaibornWolff/dcos-thanos/issues",
+  "postUninstallNotes": "DC/OS thanos is being uninstalled."
 }

utils/grafana-loader/src/grafana.rs

@@ -13,13 +13,24 @@ pub struct Dashboard {
     pub overwrite: bool
 }
 
+
 #[derive(Serialize, Deserialize)]
 struct Folder {
     pub title: String,
     pub uid: String,
     pub id: u32,
 }
 
+
+#[derive(Serialize, Deserialize)]
+struct SearchResult {
+    pub title: String,
+    pub uid: String,
+    pub id: u32,
+    pub r#type: String,
+}
+
+
 #[derive(Serialize, Deserialize)]
 struct FolderCreateInfo {
     pub title: String,
@@ -93,6 +104,21 @@ impl GrafanaAPI {
 
     #[allow(unused_must_use)]
     pub fn clear_dashboards(&mut self) {
+        let response = self.get("/api/search").send().unwrap();
+        if response.status().is_success() {
+            let result: Vec<SearchResult> = response.json().unwrap();
+            for item in result {
+                if item.r#type == "dash-db" {
+                    println!("Deleting dashboard '{}' with id {} and uid '{}'", item.title, item.id, item.uid);
+                    // Ignore errors and continue
+                    self.delete(&format!("/api/dashboards/uid/{}", item.uid)).send();
+                }
+            }
+        }
+    }
+
+    #[allow(unused_must_use)]
+    pub fn delete_folders(&mut self) {
         let response = self.get("/api/folders").send().unwrap();
         if response.status().is_success() {
             let result: Vec<Folder> = response.json().unwrap();

utils/grafana-loader/src/main.rs

@@ -27,7 +27,7 @@ fn main() {
 }
 
 fn load_datasources(api: &mut GrafanaAPI) {
-    println!("Loading datasources");
+    println!("Configuring datasources");
     let datasource_url = std::env::var("GRAFANA_DATASOURCE_URL").unwrap();
     let datasource = Datasource{id: None, name: String::from("prometheus"), r#type: String::from("prometheus"), url: datasource_url, access: String::from("proxy"), basicAuth: false, isDefault: true};
     api.create_or_update_datasource(datasource);
@@ -38,22 +38,29 @@ fn load_dashboards(api: &mut GrafanaAPI) {
     let fetcher_url = std::env::var("FETCHER_URL").unwrap();
     let base_folder = std::env::var("FETCHER_BASE_FOLDER").unwrap_or(String::from(""));
     let grafana_clear_dashboards = std::env::var("GRAFANA_CLEAR_DASHBOARDS").unwrap_or(String::from("")).to_lowercase() == "true";
+    let grafana_delete_folders = std::env::var("GRAFANA_DELETE_FOLDERS").unwrap_or(String::from("")).to_lowercase() == "true";
 
     let client = reqwest::blocking::ClientBuilder::new().danger_accept_invalid_certs(true).build().unwrap();
     let response = client.get(&fetcher_url).send();
     if response.is_err() {
-        stop();
+        println!("Could not download dashboards file: {}", response.err().unwrap());
+        println!("Stopping here.");
+        return;
     }
     let mut response = response.unwrap();
     if !response.status().is_success() {
-        stop();
+        println!("Could not download dashboards file: HTTP Error Code {}. Response Body: \n{}", response.status().as_str(), response.text().unwrap());
+        println!("Stopping here.");
+        return;
     }
     let mut buf: Vec<u8> = vec![];
     response.copy_to(&mut buf).unwrap();
     let cursor = io::Cursor::new(buf);
 
     if let Ok(mut archive) = zip::ZipArchive::new(cursor) {
-        if grafana_clear_dashboards {
+        if grafana_delete_folders {
+            api.delete_folders();
+        } else if grafana_clear_dashboards {
             api.clear_dashboards();
         }
         for i in 0..archive.len() {
@@ -70,10 +77,12 @@ fn load_dashboards(api: &mut GrafanaAPI) {
                 } else {
                     println!("Could not parse {}/{}, ignoring it", folder.to_str().unwrap(), filename.to_str().unwrap());
                 }
+            } else {
+                println!("Ignoring file {}", outpath.to_str().unwrap_or("!!!invalid!!!"));
             }
         }
     } else {
-        stop();
+        println!("Downloaded file is not a valid zip file. Stopping here");
     }
 }
 
@@ -120,9 +129,3 @@ fn determine_folder_id(api: &mut GrafanaAPI, folder: String) -> u32 {
         api.get_or_create_folder(folder)
     }
 }
-
-
-fn stop() {
-    println!("Could not download dashboards. Stopping here");
-    std::process::exit(0);
-}