Finished release 7.0.0_test.

.clocignore

@@ -0,0 +1,9 @@
+Products/ZenUI3/browser/resources/js/dagre
+Products/ZenUI3/browser/resources/js/codemirror
+Products/ZenUI3/browser/resources/css/dagre
+Products/ZenUtils/js/MochiKit.js
+Products/ZenWidgets/skins/zenui/yui
+Products/ZenWidgets/skins/zenui/yowl
+Products/ZenUI3/browser/resources/js/jasmine
+Products/ZenUI3/browser/resources/js/timezone
+Products/ZenUI3/browser/resources/js/swfobject

.gitignore

@@ -1,6 +1,11 @@
+Products/LDAPMultiPlugins/
+Products/LDAPUserFolder/
+Products/MemcachedManager/
 Products/README.txt
+Products/ZenModel/ZMigrateVersion.py
 Products/ZenUI3/browser/resources/js/deploy/
-
+Products/ZenUI3/node_modules
+bin/zensocket
 #daemons
 bin/runzope
 bin/runzope.bat
@@ -20,9 +25,11 @@ bin/zenvmwareevents
 bin/zenvmwaremodeler
 bin/zenvmwareperf
 
+dist
 
 .idea/
 *.iml
 *.pyc
 *~
-
+*.tar.gz
+*.swp

Products/DataCollector/ApplyDataMap.py

@@ -10,6 +10,7 @@
 
 import sys
 from collections import defaultdict
+
 import logging
 log = logging.getLogger("zen.ApplyDataMap")
 
@@ -18,8 +19,11 @@
 from ZODB.transact import transact
 from zope.event import notify
 from zope.container.contained import ObjectMovedEvent
+from zope.component import createObject
 from Acquisition import aq_base
+from metrology.registry import registry
 
+from Products.ZenUtils.MetricReporter import QueueGauge
 from Products.ZenUtils.Utils import importClass
 from Products.Zuul.catalog.events import IndexingEvent
 from Products.ZenUtils.events import pausedAndOptimizedIndexing
@@ -52,7 +56,6 @@ def isSameData(x, y):
 
     return x == y
 
-
 class ApplyDataMap(object):
 
     def __init__(self, datacollector=None):
@@ -61,6 +64,11 @@ def __init__(self, datacollector=None):
         self._dmd = None
         if datacollector:
             self._dmd = getattr(datacollector, 'dmd', None)
+        metricName = 'applyDataMap.updateRelationship'
+        if metricName not in {x[0] for x in registry}:
+            registry.add(metricName, QueueGauge('zenoss_deviceId', 'zenoss_compname', 'internal'))
+        self._urGauge = registry.get(metricName)
+        self.zing_datamap_handler = createObject("ZingDatamapHandler", self._dmd)
 
     def logChange(self, device, compname, eventClass, msg):
         if not getattr(device, 'zCollectorLogChanges', True): return
@@ -103,7 +111,6 @@ def applyDataMap(self, device, datamap, relname="", compname="", modname="", par
             datamap = ObjectMap(datamap, compname=compname, modname=modname)
         self._applyDataMap(device, datamap)
 
-
     def setDeviceClass(self, device, deviceClass=None):
         """
         If a device class has been passed and the current class is not /Classifier
@@ -112,9 +119,22 @@ def setDeviceClass(self, device, deviceClass=None):
         if deviceClass and device.getDeviceClassPath().startswith(CLASSIFIER_CLASS):
             device.changeDeviceClass(deviceClass)
 
+    def _applyDataMap(self, device, datamap, commit=True):
+        """Apply datamap to device. Return True if datamap changed device.
 
-    @transact
-    def _applyDataMap(self, device, datamap):
+        The default value for commit is True for backwards-compatibility
+        reasons. If you're a new caller to ApplyDataMap._applyData you should
+        probably set commit to False and handle your own transactions.
+
+        """
+        self.zing_datamap_handler.add_datamap(device, datamap)
+        if commit:
+            result = transact(self._applyDataMapImpl)(device, datamap)
+        else:
+            result = self._applyDataMapImpl(device, datamap)
+        return result
+
+    def _applyDataMapImpl(self, device, datamap):
         """Apply a datamap to a device.
         """
         self.num_obj_changed=0;
@@ -169,26 +189,36 @@ def _applyDataMap(self, device, datamap):
             else:
                 tobj = device
 
-            # Delay indexing until the map has been fully processed
-            # so we index the minimum amount
-            with pausedAndOptimizedIndexing():
-                if hasattr(datamap, "relname"):
-                    logname=datamap.relname
-                    changed = self._updateRelationship(tobj, datamap)
-                elif hasattr(datamap, 'modname'):
-                    logname=datamap.compname
-                    changed = self._updateObject(tobj, datamap)
+            if hasattr(datamap, "relname"):
+                logname = datamap.relname
+                # a 'naked' ObjectMap can have a relname but is not iterable
+                maps = datamap if hasattr(datamap, 'maps') else [datamap]
+                for objmap in maps:
+                    if objmap.modname:
+                        with self._urGauge(device.id, objmap.modname, False):
+                            changed = self._updateRelationship(tobj, datamap)
+                        break
                 else:
-                    log.warn("plugin returned unknown map skipping")
+                    changed = self._updateRelationship(tobj, datamap)
+            elif hasattr(datamap, 'modname'):
+                logname = datamap.compname
+                changed = self._updateObject(tobj, datamap)
+            else:
+                log.warn("plugin returned unknown map skipping")
 
-        if not changed:
-            transaction.abort()
-        else:
+        if changed:
             device.setLastChange()
-            trans = transaction.get()
-            trans.setUser("datacoll")
-            trans.note("data applied from automated collection")
-        log.debug("_applyDataMap for Device %s will modify %d objects for %s", device.getId(), self.num_obj_changed,logname)
+            # This is for IpInterfaces so the device get its paths updated.
+            # Should we think of doing this differently?
+            #
+            notify(IndexingEvent(device))
+
+        log.debug(
+            "_applyDataMap for Device %s will modify %d objects for %s",
+            device.getId(),
+            self.num_obj_changed,
+            logname)
+
         return changed
 
 
@@ -382,6 +412,11 @@ def _updateObject(self, obj, objmap):
         else:
             obj._p_deactivate()
         self.num_obj_changed += 1 if changed else 0
+
+        # Store the object with the objmap as the key, so when we serialize
+        # objmap, we can look up the associated UUID and other information.
+        self.zing_datamap_handler.add_context(objmap, obj)
+
         return changed
 
 

Products/DataCollector/CommandParsers/Linux_netstat_an.py

@@ -20,7 +20,7 @@
 
 class Linux_netstat_an(CommandParser):
 
-    command = 'netstat -an | grep 0.0.0.0:*'
+    command = 'netstat -ln | grep 0.0.0.0:*'
 
     def condition(self, device, log):
         pp = device.getPrimaryPath()
@@ -56,4 +56,4 @@ def parse(self, device, results, log):
         return rm
 
     def description(self):
-        return "run netstat -an on server to build ipservices"
+        return "run netstat -ln on server to build ipservices"

Products/DataCollector/ProcessCommandPlugin.py

@@ -39,10 +39,10 @@ def process(self, device, results, log):
             log.error("Unable to get data for %s -- skipping model",
                       device.id)
             return None
-
+	results = unicode(results, errors="replace")  # without relying on "ps" command output
         psOutputLines = self._filterLines(results.splitlines())
 
-        cmds = map(lambda(s):s.strip(), psOutputLines)
+        cmds = map(lambda s: s.strip(), psOutputLines)
         cmds = filter(lambda(s):s, cmds)
         rm = self.relMap()
         matchData = device.osProcessClassMatchData

Products/DataCollector/PythonClient.py

@@ -78,7 +78,7 @@ def collectComplete(self, r, plugin):
             return
 
         if isinstance(r, Failure):
-            log.warn("Error in %s: %s", plugin.name(), r.getErrorMessage())
+            log.error("Error in %s: %s", plugin.name(), r.getErrorMessage())
         else:
             log.debug("Results for %s: %s", plugin.name(), str(r))
             self.results.append((plugin, r))

Products/DataCollector/SnmpClient.py

@@ -109,10 +109,10 @@ def doRun(self, driver):
                 yield self.findSnmpCommunity()
                 snmp_config = driver.next()
                 if not snmp_config:
-                    log.warn(
+                    log.error(
                         'Failed to rediscover the SNMP connection info for %s',
                         self.device.manageIp)
-                    return
+                    raise
                 if snmp_config.version:
                     self.connInfo.zSnmpVer = snmp_config.version
                 if snmp_config.port:
@@ -122,13 +122,13 @@ def doRun(self, driver):
                 self.connInfo.changed = True
                 self.initSnmpProxy()
             else:
-                return
+                raise
         except Snmpv3Error:
-            log.info("Cannot connect to SNMP agent: {0}".format(self.connInfo.summary()))
-            return
+            log.error("Cannot connect to SNMP agent: {0}".format(self.connInfo.summary()))
+            raise
         except Exception:
             log.exception("Unable to talk: " + self.connInfo.summary())
-            return
+            raise
 
         changed = True
         # FIXME: cleanup --force option #2660
@@ -202,10 +202,11 @@ def collect(self, driver):
                                               maxRepetitions=maxRepetitions,
                                               limit=sys.maxint)
                     self._tabledata[pname][tmap] = driver.next()
-            except Exception, ex:
-                if not isinstance( ex, error.TimeoutError ):
-                    log.exception("device %s plugin %s unexpected error",
-                                  self.hostname, pname)
+            except error.TimeoutError:
+                log.error("%s %s SNMP timeout", self.device.id, pname)
+            except Exception:
+                log.exception("device %s plugin %s unexpected error",
+                              self.hostname, pname)
 
 
     def getResults(self):
@@ -232,18 +233,23 @@ def clientFinished(self, result):
         if isinstance(result, failure.Failure):
             from twisted.internet import error
             if isinstance(result.value, error.TimeoutError):
-                log.warning("Device %s timed out: are "
+                log.error("Device %s timed out: are "
                             "your SNMP settings correct?", self.hostname)
                 summary = "SNMP agent down - no response received"
                 log.info("Sending event: %s", summary)
-                self._sendStatusEvent(summary, eventKey='agent_down')
             elif isinstance(result.value, Snmpv3Error):
-                log.warning("Connection to device {0.hostname} failed: {1.value.message}".format(self, result))
+                log.error("Connection to device {0.hostname} failed: {1.value.message}".format(self, result))
+                summary = "SNMP v3 specific error during SNMP collection"
             else:
                 log.exception("Device %s had an error: %s",self.hostname,result)
+                summary = "Exception during SNMP collection"
+            self._sendStatusEvent(summary, eventKey='agent_down')
         else:
             self._sendStatusEvent('SNMP agent up', eventKey='agent_down', severity=Event.Clear)
-        self.proxy.close()
+        try:
+            self.proxy.close()
+        except AttributeError:
+            log.info("Caught AttributeError closing SNMP connection.")
         """tell the datacollector that we are all done"""
         if self.datacollector:
             self.datacollector.clientFinished(self)
@@ -252,4 +258,3 @@ def clientFinished(self, result):
 
     def stop(self):
         self.proxy.close()
-

Products/DataCollector/SshClient.py

@@ -97,7 +97,7 @@ def hasattr_path( object_root, path ):
     }
     if event_key:
         error_event['eventKey'] = event_key
-
+        
     # At this point, we don't know what we have
     try:
         if hasattr_path( self, "factory.datacollector.sendEvent" ):
@@ -356,6 +356,12 @@ def _handleFailure(self, message, event_key=None):
         log.error( message )
         sendEvent( self, message=message, event_key=event_key )
 
+    def _handleLoginSuccess(self, message, event_key=None):
+        """
+        Handle SSH login success by clearing any failure events.
+        """
+        sendEvent(self, message=message, event_key=event_key, severity=Event.Clear)
+
     def _getKey(self):
         keyPath = os.path.expanduser(self.factory.keyPath)
         log.debug('Expanded SSH key path from zKeyPath %s to %s' % (
@@ -392,7 +398,7 @@ def getPublicKey(self):
         # TODO: Would be good to expand to support sending multiple keys.
         if self._key is not None and not self._sent_pk:
             self._sent_pk = True
-            return self._key.blob()
+            return self._key
 
     def getPrivateKey(self):
         """
@@ -401,11 +407,7 @@ def getPrivateKey(self):
         @return: Twisted deferred object (defer.succeed)
         @rtype: Twisted deferred object
         """
-        if self._key is None:
-            keyObject = None
-        else:
-            keyObject = self._key.keyObject
-        return defer.succeed(keyObject)
+        return defer.succeed(self._key)
 
     def auth_keyboard_interactive(self, *args, **kwargs):
         # Don't authenticate multiple times with same credentials
@@ -441,8 +443,10 @@ def serviceStopped(self, *args, **kwargs):
             self._handleFailure(msg, event_key="sshClientAuth")
             self.factory.clientFinished(LoginFailed(msg))
         else:
-            sendEvent(self, "Authentication succeeded for username %s" % self.user, severity=Event.Clear,
+            msg = "SSH Authentication succeeded for username %s" % self.user
+            sendEvent(self, msg, severity=Event.Clear,
                       event_key="sshClientAuth")
+            self._handleLoginSuccess(msg, event_key="sshClientAuth")
         return userauth.SSHUserAuthClient.serviceStopped(self, *args, **kwargs)
 
 class SshConnection(connection.SSHConnection):