feat: capacity cronjobs

roles/yoda_report/files/gather-capacity-stats.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+"""This script gathers disk usage data for all unixfilesystem vault volumes,
+   so that it can be processed by capacity management scripts."""
+
+import os
+import socket
+import subprocess
+import yaml
+
+def get_ufs_resources(hostname):
+    """Returns dictionary with unix file system resources on a particular resource host.
+       Keys are resource names. Values are dictionaries with
+       parent_resource_number (could be None), vault"""
+    result = {}
+    current_data = {}
+
+    def _flush_current_data_if_complete():
+        if ( current_data.get("type", None) in ["unixfilesystem", "unix file system"] and
+             current_data.get("location", None) == hostname ):
+            if ( "resource name" in current_data and
+                 "vault" in current_data and
+                 "type" in current_data  ):
+                result[ current_data["resource name"] ] = {
+                    "parent_resource_number" : current_data.get("parent", None),
+                    "vault" : current_data["vault"]
+                }
+        current_data.clear()
+
+
+    for line in _get_cmd_stdout_lines(['ilsresc', '-l']):
+        if line == "----":
+            _flush_current_data_if_complete()
+        elif ( line.startswith("resource name:") or
+               line.startswith("parent:") or
+               line.startswith("vault:") or
+               line.startswith("location:") or
+               line.startswith("type:") ):
+            ( attr, value ) = line.split(": ")
+            if attr == "parent" and value == "":
+                current_data[attr] = None
+            else:
+                current_data[attr] = value
+
+    _flush_current_data_if_complete()      
+    return result
+
+
+def _get_cmd_stdout_lines(args):
+    return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.decode('UTF-8').split('\n')
+
+
+def get_diskspace_free(directory):
+    volumedata = os.statvfs(directory)
+    return volumedata.f_frsize * volumedata.f_bfree
+
+
+def get_diskspace_total(directory):
+    volumedata = os.statvfs(directory)
+    return volumedata.f_frsize * volumedata.f_blocks
+
+
+def collect_usage_data(resources):
+    return { resource_name : 
+        { "resource_in_tree" : "no" if resources[resource_name]["parent_resource_number"] is None else "yes",
+          "diskspace_free" : get_diskspace_free(resources[resource_name]["vault"]),
+          "diskspace_total" : get_diskspace_total(resources[resource_name]["vault"]) }
+        for resource_name in resources
+    }
+
+
+def write_usage_data(data, outputfile):
+    with open (outputfile, "w") as out:
+        yaml.dump(data, out)
+
+
+# Main
+hostname = socket.getfqdn()
+outputfile = "/var/yoda-financial-data/{}.capacity.yaml".format(hostname.split(".")[0])
+resources = get_ufs_resources(hostname)
+usage_data = collect_usage_data(resources)
+write_usage_data(usage_data, outputfile)

roles/yoda_report/tasks/main.yml

@@ -38,7 +38,7 @@
   when: enable_yoda_report
 
 
-- name: Copy file to remote /var directory
+- name: Copy file for gathering group stats to remote /var directory
   ansible.builtin.copy:
     src: gather-group-statistics.py
     dest: /var/yoda-financial-data/gather-group-statistics.py
@@ -48,6 +48,16 @@
   when: enable_yoda_report
 
 
+- name: Copy file for gathering capacity stats to remote /var directory
+  ansible.builtin.copy:
+    src: gather-capacity-stats.py
+    dest: /var/yoda-financial-data/gather-capacity-stats.py
+    owner: "{{ irods_service_account }}"
+    group: "{{ irods_service_account }}"
+    mode: '0700'
+  when: enable_yoda_report
+
+
 - name: Copy file to remote /var directory
   ansible.builtin.copy:
     src: requirements.txt
@@ -98,3 +108,29 @@
       {{ yoda_report_server_user_account }}@{{ yoda_report_server_host }}:{{ yoda_report_server_dir }}/{{ instance }}.stats.yml
     user: "{{ irods_service_account }}"
     state: "{{ 'present' if enable_yoda_report else 'absent' }}"
+
+
+- name: Add cron job to gather capacity statistics
+  ansible.builtin.cron:
+    name: "yoda-report-gather-capacity-statistics"
+    minute: "*/2"
+    hour: "*"
+    job: >
+      source /var/yoda-financial-data/yoda_report_venv/bin/activate &&
+      python /var/yoda-financial-data/gather-capacity-stats.py
+    user: "{{ irods_service_account }}"
+    state: "{{ 'present' if enable_yoda_report else 'absent' }}"
+
+
+- name: Add cron job to transfer capacity statistics to reporting server
+  ansible.builtin.cron:
+    name: "yoda-report-copy-capacity-statistics"
+    minute: "*/5"
+    hour: "*"
+    job: >
+      source /var/yoda-financial-data/yoda_report_venv/bin/activate &&
+      scp -i {{ yoda_report_server_priv_key_loc }}
+      /var/yoda-financial-data/*.capacity.yaml
+      {{ yoda_report_server_user_account }}@{{ yoda_report_server_host }}:{{ yoda_report_server_dir }}/{{ instance }}.capacity.stats.yml
+    user: "{{ irods_service_account }}"
+    state: "{{ 'present' if enable_yoda_report else 'absent' }}"