Enable overriding interface counters OIDs (sonic-net#98)

* Add test for interface IN_OCTETS
* Add OverlayAdpaterMIBEntry class
* Refine
* InterfacesMIB supports counter override
* (reformat)
* Add missing test data
* Remvoe unused code
* Refactor
* (reformat)
* InterfaceMIBObjects supports counter override

src/ax_interface/mib.py

@@ -86,22 +86,16 @@ def __new__(mcs, name, bases, attributes, prefix=None):
             for me in vars(cls).values():
                 if isinstance(me, MIBEntry):
                     setattr(me, MIBEntry.PREFIXLEN, _prefix_len + len(me.subtree))
+                    setattr(me, MIBEntry.PREFIX, _prefix + me.subtree)
 
             sub_ids = {}
 
-            # gather all static MIB entries.
-            static_entries = (v for v in vars(cls).values() if type(v) is MIBEntry)
-            for me in static_entries:
+            # gather all MIB entries.
+            mib_entries = (v for v in vars(cls).values() if isinstance(v, MIBEntry))
+            for me in mib_entries:
                 sub_ids.update({_prefix + me.subtree: me})
                 prefixes.append(_prefix + me.subtree)
 
-            # gather all subtree IDs
-            # to support dynamic sub_id in the subtree, not to pour leaves into dictionary
-            subtree_entries = (v for v in vars(cls).values() if type(v) is SubtreeMIBEntry)
-            for sme in subtree_entries:
-                sub_ids.update({_prefix + sme.subtree: sme})
-                prefixes.append(_prefix + sme.subtree)
-
             # gather all updater instances
             updaters = set(v for k, v in vars(cls).items() if isinstance(v, MIBUpdater))
 
@@ -134,6 +128,7 @@ def __init__(cls, name, bases, attributes, prefix=None):
 
 class MIBEntry:
     PREFIXLEN = '__prefixlen__'
+    PREFIX = '__prefix__'
 
     def __init__(self, subtree, value_type, callable_, *args):
         """
@@ -155,7 +150,7 @@ def __init__(self, subtree, value_type, callable_, *args):
             raise ValueError("Third argument must be a callable object--got literal instead.")
         self._callable_ = callable_
         self._callable_args = args
-        self.subtree = subtree
+        self.subtree_str = subtree
         self.value_type = value_type
         self.subtree = util.oid2tuple(subtree, dot_prefix=False)
 
@@ -174,8 +169,11 @@ def replace_sub_id(self, oid_key, sub_id):
     def get_next(self, sub_id):
         return None
 
+    def get_prefix(self):
+        return getattr(self, MIBEntry.PREFIX)
+
 class SubtreeMIBEntry(MIBEntry):
-    def __init__(self, subtree, iterator, value_type, callable_, *args, updater=None):
+    def __init__(self, subtree, iterator, value_type, callable_, *args):
         super().__init__(subtree, value_type, callable_, *args)
         self.iterator = iterator
 
@@ -209,6 +207,46 @@ def get_next(self, sub_id):
             logger.exception("SubtreeMIBEntry.get_next() caught an unexpected exception during iterator.get_next()")
             return None
 
+# Define MIB entry (subtree) with a callable, which accepts a starndard OID tuple as a paramter
+class OidMIBEntry(MIBEntry):
+    def __init__(self, subtree, value_type, callable_):
+        super().__init__(subtree, value_type, callable_)
+
+    def __iter__(self):
+        raise NotImplementedError
+
+    def __call__(self, sub_id):
+        return self._callable_.__call__(self.get_prefix() + sub_id)
+
+class OverlayAdpaterMIBEntry(MIBEntry):
+    def __init__(self, underlay_mibentry, overlay_mibentry):
+        assert underlay_mibentry.value_type == overlay_mibentry.value_type
+        assert underlay_mibentry.subtree == overlay_mibentry.subtree
+
+        super().__init__(underlay_mibentry.subtree_str, underlay_mibentry.value_type, underlay_mibentry._callable_)
+        self.underlay_mibentry = underlay_mibentry
+        self.overlay_mibentry = overlay_mibentry
+
+    def __setattr__(self, name, value):
+        super().__setattr__(name, value)
+        if name.startswith('__') and name.endswith('__'):
+            setattr(self.underlay_mibentry, name, value)
+            setattr(self.overlay_mibentry, name, value)
+
+    def __iter__(self):
+        return self.underlay_mibentry.__iter__()
+
+    def __call__(self, sub_id=None):
+        overlay_val = self.overlay_mibentry(sub_id)
+        if overlay_val is not None:
+            return overlay_val
+
+        underlay_val = self.underlay_mibentry(sub_id)
+        return underlay_val
+
+    def get_next(self, sub_id):
+        return self.underlay_mibentry.get_next(sub_id)
+
 class MIBTable(dict):
     """
     Simplistic LUT for Get/GetNext OID. Interprets iterables as keys and implements the same interfaces as dict's.

src/sonic_ax_impl/mibs/__init__.py

@@ -4,6 +4,8 @@
 from swsssdk import SonicV2Connector
 from swsssdk import port_util
 from swsssdk.port_util import get_index, get_index_from_str
+from ax_interface.mib import MIBUpdater
+from ax_interface.util import oid2tuple
 from sonic_ax_impl import logger
 
 COUNTERS_PORT_NAME_MAP = b'COUNTERS_PORT_NAME_MAP'
@@ -16,6 +18,7 @@
 COUNTERS_DB = 'COUNTERS_DB'
 CONFIG_DB = 'CONFIG_DB'
 STATE_DB = 'STATE_DB'
+SNMP_OVERLAY_DB = 'SNMP_OVERLAY_DB'
 
 TABLE_NAME_SEPARATOR_COLON = ':'
 TABLE_NAME_SEPARATOR_VBAR = '|'
@@ -363,3 +366,56 @@ def get_transceiver_sensor_sub_id(ifindex, sensor):
 
     transceiver_oid, = get_transceiver_sub_id(ifindex)
     return (transceiver_oid + SENSOR_PART_ID_MAP[sensor], )
+
+class RedisOidTreeUpdater(MIBUpdater):
+    def __init__(self, prefix_str):
+        super().__init__()
+
+        self.db_conn = init_db()
+        if prefix_str.startswith('.'):
+            prefix_str = prefix_str[1:]
+        self.prefix_str = prefix_str
+
+    def get_next(self, sub_id):
+        """
+        :param sub_id: The 1-based sub-identifier query.
+        :return: the next sub id.
+        """
+        raise NotImplementedError
+
+    def reinit_data(self):
+        """
+        Subclass update loopback information
+        """
+        pass
+
+    def update_data(self):
+        """
+        Update redis (caches config)
+        Pulls the table references for each interface.
+        """
+        self.oid_list = []
+        self.oid_map = {}
+
+        self.db_conn.connect(SNMP_OVERLAY_DB)
+        keys = self.db_conn.keys(SNMP_OVERLAY_DB, self.prefix_str + '*')
+        # TODO: fix db_conn.keys to return empty list instead of None if there is no match
+        if keys is None:
+            keys = []
+
+        for key in keys:
+            key = key.decode()
+            oid = oid2tuple(key, dot_prefix=False)
+            self.oid_list.append(oid)
+            value = self.db_conn.get_all(SNMP_OVERLAY_DB, key)
+            if value[b'type'] in [b'COUNTER_32', b'COUNTER_64']:
+                self.oid_map[oid] = int(value[b'data'])
+            else:
+                raise ValueError("Invalid value type")
+
+        self.oid_list.sort()
+
+    def get_oidvalue(self, oid):
+        if oid not in self.oid_map:
+            return None
+        return self.oid_map[oid]

src/sonic_ax_impl/mibs/ietf/rfc1213.py

@@ -4,11 +4,10 @@
 from bisect import bisect_right
 
 from sonic_ax_impl import mibs
-from ax_interface import MIBMeta, ValueType, MIBUpdater, MIBEntry, SubtreeMIBEntry
+from ax_interface.mib import MIBMeta, ValueType, MIBUpdater, MIBEntry, SubtreeMIBEntry, OverlayAdpaterMIBEntry, OidMIBEntry
 from ax_interface.encodings import ObjectIdentifier
 from ax_interface.util import mac_decimals, ip2tuple_v4
 
-
 @unique
 class DbTables(int, Enum):
     """
@@ -271,6 +270,7 @@ def get_counter(self, sub_id, table_name):
         :param table_name: the redis table (either IntEnum or string literal) to query.
         :return: the counter for the respective sub_id/table.
         """
+
         oid = self.get_oid(sub_id)
         if not oid:
             return
@@ -423,14 +423,15 @@ def get_if_type(self, sub_id):
         else:
             return IfTypes.ethernetCsmacd
 
-
 class InterfacesMIB(metaclass=MIBMeta, prefix='.1.3.6.1.2.1.2'):
     """
     'interfaces' https://tools.ietf.org/html/rfc1213#section-3.5
     """
 
     if_updater = InterfacesUpdater()
 
+    oidtree_updater = mibs.RedisOidTreeUpdater(prefix_str='1.3.6.1.2.1.2')
+
     # (subtree, value_type, callable_, *args, handler=None)
     ifNumber = MIBEntry('1', ValueType.INTEGER, if_updater.get_if_number)
 
@@ -467,52 +468,88 @@ class InterfacesMIB(metaclass=MIBMeta, prefix='.1.3.6.1.2.1.2'):
         SubtreeMIBEntry('2.1.9', if_updater, ValueType.TIME_TICKS, lambda sub_id: 0)
 
     ifInOctets = \
-        SubtreeMIBEntry('2.1.10', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(10))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.10', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(10)),
+            OidMIBEntry('2.1.10', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifInUcastPkts = \
-        SubtreeMIBEntry('2.1.11', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(11))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.11', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(11)),
+            OidMIBEntry('2.1.11', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifInNUcastPkts = \
-        SubtreeMIBEntry('2.1.12', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(12))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.12', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(12)),
+            OidMIBEntry('2.1.12', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifInDiscards = \
-        SubtreeMIBEntry('2.1.13', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(13))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.13', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(13)),
+            OidMIBEntry('2.1.13', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifInErrors = \
-        SubtreeMIBEntry('2.1.14', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(14))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.14', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(14)),
+            OidMIBEntry('2.1.14', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifInUnknownProtos = \
-        SubtreeMIBEntry('2.1.15', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(15))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.15', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(15)),
+            OidMIBEntry('2.1.15', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutOctets = \
-        SubtreeMIBEntry('2.1.16', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(16))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.16', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(16)),
+            OidMIBEntry('2.1.16', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutUcastPkts = \
-        SubtreeMIBEntry('2.1.17', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(17))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.17', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(17)),
+            OidMIBEntry('2.1.17', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutNUcastPkts = \
-        SubtreeMIBEntry('2.1.18', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(18))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.18', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(18)),
+            OidMIBEntry('2.1.18', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutDiscards = \
-        SubtreeMIBEntry('2.1.19', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(19))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.19', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(19)),
+            OidMIBEntry('2.1.19', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutErrors = \
-        SubtreeMIBEntry('2.1.20', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
-                           DbTables(20))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.20', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+                           DbTables(20)),
+            OidMIBEntry('2.1.20', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+        )
 
     ifOutQLen = \
-        SubtreeMIBEntry('2.1.21', if_updater, ValueType.GAUGE_32, if_updater.get_counter,
-                           DbTables(21))
+        OverlayAdpaterMIBEntry(
+            SubtreeMIBEntry('2.1.21', if_updater, ValueType.GAUGE_32, if_updater.get_counter,
+                           DbTables(21)),
+            OidMIBEntry('2.1.21', ValueType.GAUGE_32, oidtree_updater.get_oidvalue)
+        )
 
     # FIXME Placeholder
     ifSpecific = \