Add unit conversion for metering views

Until now, the charts displayed by the metering views (Admin -> Resource
Usage) were using units as defined by the Ceilometer meters. This means
that the values in the charts were sometimes very hard to read for
humans (typically we would see a large number in nanoseconds, bytes,
etc).

This patch introduces support for normalization of these values. The
statistics returned by the Ceilometer API are analyzed to determine
the appropriate unit for the series. The whole series is then
converted to this unit, so that the values in the series are more
easily parsed by humans.

Implements blueprint: ceilometer-api-statistics-decorators-and-convertors
Change-Id: Ia183ca5270ebd1dcbd1fe6ac20ed4dfe4d6ebcc4

horizon/static/horizon/js/horizon.d3linechart.js

@@ -477,6 +477,12 @@ horizon.d3_line_chart = {
         var hoverDetail = new Rickshaw.Graph.HoverDetail({
           graph: graph,
           formatter: function(series, x, y) {
+            if(y % 1 === 0) {
+              y = parseInt(y);
+            } else {
+              y = parseFloat(y).toFixed(2);
+            }
+
             var d = new Date(x * 1000);
             // Convert datetime to YYYY-MM-DD HH:MM:SS GMT
             var datetime_string = d.getUTCFullYear() + "-" +
@@ -488,8 +494,7 @@ horizon.d3_line_chart = {
 
             var date = '<span class="date">' + datetime_string + '</span>';
             var swatch = '<span class="detail_swatch" style="background-color: ' + series.color + '"></span>';
-            var content = swatch + series.name + ': ' + parseFloat(y).toFixed(2) + ' ' + series.unit + '<br>' + date;
-            return content;
+            return swatch + series.name + ': ' + y + ' ' + series.unit + '<br>' + date;
           }
         });
       }

horizon/test/tests/utils.py

@@ -27,6 +27,7 @@
 from horizon.utils import functions
 from horizon.utils import memoized
 from horizon.utils import secret_key
+from horizon.utils import units
 from horizon.utils import validators
 
 
@@ -395,3 +396,62 @@ def test_bad_default_value(self):
         self.assertRaises(ValueError,
                           functions.get_page_size,
                           request, default)
+
+
+class UnitsTests(test.TestCase):
+    def test_is_supported(self):
+        self.assertTrue(units.is_supported('MB'))
+        self.assertTrue(units.is_supported('min'))
+        self.assertFalse(units.is_supported('KWh'))
+        self.assertFalse(units.is_supported('unknown_unit'))
+
+    def test_is_larger(self):
+        self.assertTrue(units.is_larger('KB', 'B'))
+        self.assertTrue(units.is_larger('MB', 'B'))
+        self.assertTrue(units.is_larger('GB', 'B'))
+        self.assertTrue(units.is_larger('TB', 'B'))
+        self.assertTrue(units.is_larger('GB', 'MB'))
+        self.assertFalse(units.is_larger('B', 'KB'))
+        self.assertFalse(units.is_larger('MB', 'GB'))
+
+        self.assertTrue(units.is_larger('min', 's'))
+        self.assertTrue(units.is_larger('hr', 'min'))
+        self.assertTrue(units.is_larger('hr', 's'))
+        self.assertFalse(units.is_larger('s', 'min'))
+
+    def test_convert(self):
+        self.assertEqual(units.convert(4096, 'MB', 'GB'), (4, 'GB'))
+        self.assertEqual(units.convert(4, 'GB', 'MB'), (4096, 'MB'))
+
+        self.assertEqual(units.convert(1.5, 'hr', 'min'), (90, 'min'))
+        self.assertEqual(units.convert(12, 'hr', 'day'), (0.5, 'day'))
+
+    def test_normalize(self):
+        self.assertEqual(units.normalize(1, 'B'), (1, 'B'))
+        self.assertEqual(units.normalize(1000, 'B'), (1000, 'B'))
+        self.assertEqual(units.normalize(1024, 'B'), (1, 'KB'))
+        self.assertEqual(units.normalize(1024 * 1024, 'B'), (1, 'MB'))
+        self.assertEqual(units.normalize(10 * 1024 ** 3, 'B'), (10, 'GB'))
+        self.assertEqual(units.normalize(1000 * 1024 ** 4, 'B'), (1000, 'TB'))
+        self.assertEqual(units.normalize(1024, 'KB'), (1, 'MB'))
+        self.assertEqual(units.normalize(1024 ** 2, 'KB'), (1, 'GB'))
+        self.assertEqual(units.normalize(10 * 1024, 'MB'), (10, 'GB'))
+        self.assertEqual(units.normalize(0.5, 'KB'), (512, 'B'))
+        self.assertEqual(units.normalize(0.0001, 'MB'), (104.9, 'B'))
+
+        self.assertEqual(units.normalize(1, 's'), (1, 's'))
+        self.assertEqual(units.normalize(60, 's'), (1, 'min'))
+        self.assertEqual(units.normalize(3600, 's'), (1, 'hr'))
+        self.assertEqual(units.normalize(3600 * 24, 's'), (1, 'day'))
+        self.assertEqual(units.normalize(10 * 3600 * 24, 's'), (1.4, 'week'))
+        self.assertEqual(units.normalize(60, 'min'), (1, 'hr'))
+        self.assertEqual(units.normalize(90, 'min'), (1.5, 'hr'))
+        self.assertEqual(units.normalize(60 * 24, 'min'), (1, 'day'))
+        self.assertEqual(units.normalize(0.5, 'day'), (12, 'hr'))
+        self.assertEqual(units.normalize(10800000000000, 'ns'), (3, 'hr'))
+        self.assertEqual(units.normalize(7, 'day'), (1, 'week'))
+        self.assertEqual(units.normalize(31, 'day'), (1, 'month'))
+        self.assertEqual(units.normalize(12, 'month'), (1, 'year'))
+
+        self.assertEqual(units.normalize(1, 'unknown_unit'),
+                         (1, 'unknown_unit'))

horizon/utils/functions.py

@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import decimal
 import math
 import re
 
@@ -72,3 +73,55 @@ def get_page_size(request, default=20):
 def natural_sort(attr):
     return lambda x: [int(s) if s.isdigit() else s for s in
                       re.split(r'(\d+)', getattr(x, attr, x))]
+
+
+def get_keys(tuple_of_tuples):
+    """Processes a tuple of 2-element tuples and returns a tuple containing
+    first component of each tuple.
+    """
+    return tuple([t[0] for t in tuple_of_tuples])
+
+
+def value_for_key(tuple_of_tuples, key):
+    """Processes a tuple of 2-element tuples and returns the value
+    corresponding to the given key. If not value is found, the key is returned.
+    """
+    for t in tuple_of_tuples:
+        if t[0] == key:
+            return t[1]
+    else:
+        return key
+
+
+def next_key(tuple_of_tuples, key):
+    """Processes a tuple of 2-element tuples and returns the key which comes
+    after the given key.
+    """
+    for i, t in enumerate(tuple_of_tuples):
+        if t[0] == key:
+            try:
+                return tuple_of_tuples[i + 1][0]
+            except IndexError:
+                return None
+
+
+def previous_key(tuple_of_tuples, key):
+    """Processes a tuple of 2-element tuples and returns the key which comes
+    before the given key.
+    """
+    for i, t in enumerate(tuple_of_tuples):
+        if t[0] == key:
+            try:
+                return tuple_of_tuples[i - 1][0]
+            except IndexError:
+                return None
+
+
+def format_value(value):
+    """Returns the given value rounded to one decimal place if it is a
+    decimal, or integer if it is an integer.
+    """
+    value = decimal.Decimal(str(value))
+    if int(value) == value:
+        return int(value)
+    return round(value, 1)

horizon/utils/units.py

@@ -0,0 +1,145 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import decimal
+
+import pint
+
+from horizon.utils import functions
+
+# Mapping of units from Ceilometer to Pint
+INFORMATION_UNITS = (
+    ('B', 'byte'),
+    ('KB', 'Kibyte'),
+    ('MB', 'Mibyte'),
+    ('GB', 'Gibyte'),
+    ('TB', 'Tibyte'),
+    ('PB', 'Pibyte'),
+    ('EB', 'Eibyte'),
+)
+
+TIME_UNITS = ('ns', 's', 'min', 'hr', 'day', 'week', 'month', 'year')
+
+
+ureg = pint.UnitRegistry()
+
+
+def is_supported(unit):
+    """Returns a bool indicating whether the unit specified is supported by
+    this module.
+    """
+    return unit in functions.get_keys(INFORMATION_UNITS) + TIME_UNITS
+
+
+def is_larger(unit_1, unit_2):
+    """Returns a boolean indicating whether unit_1 is larger than unit_2.
+
+    E.g:
+
+    >>> is_larger('KB', 'B')
+    True
+    >>> is_larger('min', 'day')
+    False
+    """
+    unit_1 = functions.value_for_key(INFORMATION_UNITS, unit_1)
+    unit_2 = functions.value_for_key(INFORMATION_UNITS, unit_2)
+
+    return ureg.parse_expression(unit_1) > ureg.parse_expression(unit_2)
+
+
+def convert(value, source_unit, target_unit, fmt=False):
+    """Converts value from source_unit to target_unit. Returns a tuple
+    containing the converted value and target_unit.  Having fmt set to True
+    causes the value to be formatted to 1 decimal digit if it's a decimal or
+    be formatted as integer if it's an integer.
+
+    E.g:
+
+    >>> convert(2, 'hr', 'min')
+    (120.0, 'min')
+    >>> convert(2, 'hr', 'min', fmt=True)
+    (120, 'min')
+    >>> convert(30, 'min', 'hr', fmt=True)
+    (0.5, 'hr')
+    """
+    orig_target_unit = target_unit
+    source_unit = functions.value_for_key(INFORMATION_UNITS, source_unit)
+    target_unit = functions.value_for_key(INFORMATION_UNITS, target_unit)
+
+    q = ureg.Quantity(value, source_unit)
+    q = q.to(ureg.parse_expression(target_unit))
+    value = functions.format_value(q.magnitude) if fmt else q.magnitude
+    return value, orig_target_unit
+
+
+def normalize(value, unit):
+    """Converts the value so that it belongs to some expected range.
+    Returns the new value and new unit.
+
+    E.g:
+
+    >>> normalize(1024, 'KB')
+    (1, 'MB')
+    >>> normalize(90, 'min')
+    (1.5, 'hr')
+    >>> normalize(1.0, 'object')
+    (1, 'object')
+    """
+    if value < 0:
+        raise ValueError('Negative value: %s %s.' % (value, unit))
+
+    if unit in functions.get_keys(INFORMATION_UNITS):
+        return _normalize_information(value, unit)
+    elif unit in TIME_UNITS:
+        return _normalize_time(value, unit)
+    else:
+        # Unknown unit, just return it
+        return functions.format_value(value), unit
+
+
+def _normalize_information(value, unit):
+    value = decimal.Decimal(str(value))
+
+    while value < 1:
+        prev_unit = functions.previous_key(INFORMATION_UNITS, unit)
+        if prev_unit is None:
+            break
+        value, unit = convert(value, unit, prev_unit)
+
+    while value >= 1024:
+        next_unit = functions.next_key(INFORMATION_UNITS, unit)
+        if next_unit is None:
+            break
+        value, unit = convert(value, unit, next_unit)
+
+    return functions.format_value(value), unit
+
+
+def _normalize_time(value, unit):
+    value, unit = convert(value, unit, 's')
+
+    if value >= 60:
+        value, unit = convert(value, 's', 'min')
+
+        if value >= 60:
+            value, unit = convert(value, 'min', 'hr')
+
+            if value >= 24:
+                value, unit = convert(value, 'hr', 'day')
+
+                if value >= 365:
+                    value, unit = convert(value, 'day', 'year')
+                elif value >= 31:
+                    value, unit = convert(value, 'day', 'month')
+                elif value >= 7:
+                    value, unit = convert(value, 'day', 'week')
+
+    return functions.format_value(value), unit

openstack_dashboard/dashboards/admin/metering/tabs.py

@@ -28,7 +28,7 @@
 class GlobalStatsTab(tabs.TableTab):
     name = _("Stats")
     slug = "stats"
-    template_name = ("admin/metering/stats.html")
+    template_name = "admin/metering/stats.html"
     preload = False
     table_classes = (metering_tables.UsageTable,)