Skip to content

Commit

Permalink
Remake MSTransferor alert subject; increase alert expiration time to 1h
Browse files Browse the repository at this point in the history
make it a warning log level

switch the subject back to the service name; expiry in an hour
  • Loading branch information
amaltaro committed May 6, 2021
1 parent 3ac62d2 commit 704a3e1
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions src/python/WMCore/MicroService/MSTransferor/MSTransferor.py
Original file line number Diff line number Diff line change
Expand Up @@ -746,18 +746,21 @@ def notifyLargeData(self, aboveWarningThreshold, transferId, wflowName, dataSize
"""
# Warn about data transfer subscriptions going above some threshold
if aboveWarningThreshold:
alertName = "ms-transferor: Transfer over threshold: {}".format(transferId)
alertName = "{}: input data transfer over threshold: {}".format(self.alertServiceName,
wflowName)
alertSeverity = "high"
alertSummary = "[MS] Large pending data transfer under request id: {}".format(transferId)
alertDescription = "Workflow: {}\nhas a large amount of ".format(wflowName)
alertDescription += "data subscribed: {} TB,\n".format(teraBytes(dataSize))
alertDescription = "Workflow: {} has a large amount of ".format(wflowName)
alertDescription += "data subscribed: {} TB, ".format(teraBytes(dataSize))
alertDescription += "for {} data: {}.""".format(dataIn['type'], dataIn['name'])

try:
self.alertManagerApi.sendAlert(alertName, alertSeverity, alertSummary, alertDescription, self.alertServiceName)
# alert to expiry in an hour from now
self.alertManagerApi.sendAlert(alertName, alertSeverity, alertSummary, alertDescription,
self.alertServiceName, endSecs=1 * 60 * 60)
except Exception as ex:
self.logger.exception("Failed to send alert to %s. Error: %s", self.alertManagerUrl, str(ex))
self.logger.info(alertDescription)
self.logger.warning(alertDescription)

def _getValidSites(self, wflow, dataIn):
"""
Expand Down

0 comments on commit 704a3e1

Please sign in to comment.