diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index d413b4327ad..77ee13814e0 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -97,8 +97,8 @@ jobs: git show # Create the tag git tag "$NEW_VERSION" - echo ::set-output name=create-release::true - echo ::set-output name=new-version::"$NEW_VERSION" + echo "create-release=true" >> $GITHUB_OUTPUT + echo "new-version=$NEW_VERSION" >> $GITHUB_OUTPUT fi fi fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc465f1ef30..2070f6d078a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -19,7 +19,7 @@ repos: - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 23.1.0 hooks: - id: black exclude: | @@ -44,7 +44,7 @@ repos: )$ - repo: https://github.com/asottile/pyupgrade - rev: v3.1.0 + rev: v3.3.1 hooks: - id: pyupgrade args: ["--py39-plus"] diff --git a/dirac.cfg b/dirac.cfg index 0608afeac5f..6c974508a62 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -187,7 +187,7 @@ Registry VOMSName = lhcb # Registered identity provider associated with VO - IdP = CheckIn + IdProvider = CheckIn # Section to describe all the VOMS servers that can be used with the given VOMS VO VOMSServers diff --git a/docs/diracdoctools/Utilities.py b/docs/diracdoctools/Utilities.py index fb1c31d096f..5b4fe5aed1a 100644 --- a/docs/diracdoctools/Utilities.py +++ b/docs/diracdoctools/Utilities.py @@ -57,7 +57,7 @@ def writeLinesToFile(filename, lines): except AttributeError: # ignore decode if newContent is python3 str pass - except (UnicodeDecodeError) as e: + except UnicodeDecodeError as e: LOG.error('Failed to decode newContent with "utf-8": %r', e) raise rst.write(newContent) diff --git a/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py b/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py index 9b3fc28f99e..bb2a24197c8 100755 --- a/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py +++ b/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py @@ -720,7 +720,6 @@ def createRelease(self): if __name__ == "__main__": - RUNNER = GithubInterface() try: RUNNER.parseOptions() diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py index 5655d7bbde8..cfea7cb1877 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py @@ -13,7 +13,6 @@ class PingPongMindHandler(ExecutorMindHandler): - MSG_DEFINITIONS = {"StartReaction": {"numBounces": int}} auth_msg_StartReaction = ["all"] diff --git a/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py b/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py index 14324287a7d..c1c6d2aaba3 100644 --- a/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py +++ b/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py @@ -17,7 +17,6 @@ class PingPongHandler(RequestHandler): - MSG_DEFINITIONS = {"Ping": {"id": int}, "Pong": {"id": int}} auth_conn_connected = ["all"] diff --git a/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py b/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py index e6ac8508776..4076c6c3182 100644 --- a/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py +++ b/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py @@ -29,7 +29,6 @@ class NetworkAgent(AgentModule): BUFFER_TIMEOUT = 3600 def initialize(self): - self.log = gLogger.getSubLogger(self.__class__.__name__) # API initialization is required to get an up-to-date configuration from the CS @@ -113,7 +112,6 @@ def checkConsumers(self): # recreate consumers if there are any problems if not self.consumers or self.messagesCount == self.messagesCountOld: - for consumer in self.consumers: consumer.close() diff --git a/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py b/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py index 95140f3f2ff..bcc16830b3d 100644 --- a/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py +++ b/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py @@ -36,7 +36,6 @@ class NetworkAgentSuccessTestCase(unittest.TestCase): """Test class to check success scenarios.""" def setUp(self): - # external dependencies module.datetime = MagicMock() @@ -62,7 +61,6 @@ def tearDownClass(cls): sys.modules.pop("DIRAC.AccountingSystem.Agent.NetworkAgent") def test_updateNameDictionary(self): - module.gConfig.getConfigurationTree.side_effect = [ {"OK": True, "Value": INITIAL_CONFIG}, {"OK": True, "Value": UPDATED_CONFIG}, @@ -85,7 +83,6 @@ def test_updateNameDictionary(self): self.assertRaises(KeyError, lambda: self.agent.nameDictionary[SITE2_HOST1]) def test_agentExecute(self): - module.NetworkAgent.am_getOption.return_value = f"{MQURI1}, {MQURI2}" module.gConfig.getConfigurationTree.return_value = {"OK": True, "Value": INITIAL_CONFIG} diff --git a/src/DIRAC/AccountingSystem/Client/ReportsClient.py b/src/DIRAC/AccountingSystem/Client/ReportsClient.py index f090da6885f..522b3ce2c9d 100644 --- a/src/DIRAC/AccountingSystem/Client/ReportsClient.py +++ b/src/DIRAC/AccountingSystem/Client/ReportsClient.py @@ -30,7 +30,6 @@ def listReports(self, typeName): return result def getReport(self, typeName, reportName, startTime, endTime, condDict, grouping, extraArgs=None): - if not isinstance(extraArgs, dict): extraArgs = {} plotRequest = { diff --git a/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py b/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py index 023ad56de59..b2f3e3e7f29 100644 --- a/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py +++ b/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py @@ -13,7 +13,6 @@ class TestCase(unittest.TestCase): """Base class for the AccountingDB test cases""" def setUp(self): - self.moduleTested = moduleTested self.testClass = self.moduleTested.AccountingDB diff --git a/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py b/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py index aadc73818b7..0d43a0ae515 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py @@ -16,7 +16,6 @@ class BaseReporter(DBUtils): - _PARAM_CHECK_FOR_NONE = "checkNone" _PARAM_CALCULATE_PROPORTIONAL_GAUGES = "calculateProportionalGauges" _PARAM_CONVERT_TO_GRANULARITY = "convertToGranularity" diff --git a/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py index 4b6c81d31fb..37fcc21dc04 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py @@ -4,7 +4,6 @@ class DataOperationPlotter(BaseReporter): - _typeName = "DataOperation" _typeKeyFields = [dF[0] for dF in DataOperation().definitionKeyFields] diff --git a/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py index 7aad23619b1..8e3a4d0c174 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py @@ -6,7 +6,6 @@ class JobPlotter(BaseReporter): - _typeName = "Job" _typeKeyFields = [dF[0] for dF in Job().definitionKeyFields] diff --git a/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py index 41bc38dc732..352efad8419 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py @@ -14,14 +14,12 @@ class NetworkPlotter(BaseReporter): - _typeName = "Network" _typeKeyFields = [dF[0] for dF in Network().definitionKeyFields] _reportPacketLossRateName = "Packet loss rate" def _reportPacketLossRate(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, 100 - SUM(%s)/SUM(%s), 100", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "PacketLossRate", "entriesInBucket"], @@ -43,7 +41,6 @@ def _reportPacketLossRate(self, reportRequest): return S_OK({"data": dataDict, "granularity": granularity}) def _plotPacketLossRate(self, reportRequest, plotInfo, filename): - # prepare custom scale (10,20,...,100) scale_data = dict(zip(range(0, 101), range(100, -1, -1))) scale_ticks = list(range(0, 101, 10)) @@ -63,7 +60,6 @@ def _plotPacketLossRate(self, reportRequest, plotInfo, filename): _reportMagnifiedPacketLossRateName = "Packet loss rate (magnified)" def _reportMagnifiedPacketLossRate(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, 100 - IF(SUM(%s)/SUM(%s)*10 > 100, 100, SUM(%s)/SUM(%s)*10), 100", @@ -94,7 +90,6 @@ def _reportMagnifiedPacketLossRate(self, reportRequest): return S_OK({"data": dataDict, "granularity": granularity}) def _plotMagnifiedPacketLossRate(self, reportRequest, plotInfo, filename): - # prepare custom scale (1..10, 100) boundaries = list(np.arange(0, 10, 0.1)) boundaries.extend(range(10, 110, 10)) @@ -120,7 +115,6 @@ def _plotMagnifiedPacketLossRate(self, reportRequest, plotInfo, filename): _reportAverageOneWayDelayName = "One-way delay (average)" def _reportAverageOneWayDelay(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "OneWayDelay", "entriesInBucket"], @@ -158,7 +152,6 @@ def _plotAverageOneWayDelay(self, reportRequest, plotInfo, filename): _reportJitterName = "Jitter" def _reportJitter(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "Jitter", "entriesInBucket"], @@ -196,7 +189,6 @@ def _plotJitter(self, reportRequest, plotInfo, filename): _reportJitterDelayRatioName = "Jitter/Delay" def _reportJitterDelayRatio(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "Jitter", "OneWayDelay"], diff --git a/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py index 10176ebe9f7..9e229fcb99f 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py @@ -4,7 +4,6 @@ class PilotPlotter(BaseReporter): - _typeName = "Pilot" _typeKeyFields = [dF[0] for dF in Pilot().definitionKeyFields] diff --git a/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py index f6e111a6c93..e42b55289a1 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py @@ -4,7 +4,6 @@ class WMSHistoryPlotter(BaseReporter): - _typeName = "WMSHistory" _typeKeyFields = [dF[0] for dF in WMSHistory().definitionKeyFields] diff --git a/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py b/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py index d11403fcb3d..36a6a311636 100644 --- a/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py +++ b/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py @@ -2,7 +2,6 @@ class FilterExecutor: - ALLKW = "all" def __init__(self): diff --git a/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py index b759f504ca5..854f4e88468 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py @@ -191,7 +191,6 @@ def __lookForNewCEs(self): return S_OK() def __getGlue2CEInfo(self, vo): - if vo in self.voBdiiCEDict: return S_OK(self.voBdiiCEDict[vo]) self.log.info("Check for available CEs for VO", vo) @@ -279,7 +278,6 @@ def __purgeSites(self, ceBdiiDict): return def __updateCS(self, bdiiChangeSet): - queueVODict = {} changeSet = set() for entry in bdiiChangeSet: diff --git a/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py index e3ba3622d93..879142b80c2 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py @@ -235,7 +235,6 @@ def __updateConfiguration(self, setElements=None, delElements=None): # assure existence and proper value of a section or an option for path, value in setElements.items(): - if value is None: section = path else: diff --git a/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py index 985b6454ccb..1c608282d7c 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py @@ -80,7 +80,6 @@ def initialize(self): return S_OK() def execute(self): - for vo in self.voList: voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None @@ -198,7 +197,6 @@ def __syncCSWithVOMS(self, vomsSync): @executeWithUserProxy def __addHomeDirectory(self, vo, newUsers): - fc = FileCatalog(vo=vo) defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) diff --git a/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py b/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py index 3c7de2e02f5..06de61e7329 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py +++ b/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py @@ -37,7 +37,6 @@ def tearDown(self): pass def test__getGlue2CEInfo_success(self): - expectedResult = {} expectedResult.update(ALTBDII) expectedResult.update(MAINBDII) @@ -61,7 +60,6 @@ def test__getGlue2CEInfo_success(self): self.assertNotIn("ce2b", ret["Value"]["site2"]["CEs"]) def test__getGlue2CEInfo_fail_10(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", @@ -86,7 +84,6 @@ def test__getGlue2CEInfo_fail_10(self): self.assertEqual(ALTBDII, ret["Value"]) def test__getGlue2CEInfo_fail_01(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", @@ -111,7 +108,6 @@ def test__getGlue2CEInfo_fail_01(self): self.assertEqual(MAINBDII, ret["Value"]) def test__getGlue2CEInfo_fail_11(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py index 5f40fa9fcda..191a55d5734 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py @@ -27,21 +27,21 @@ def getUsernameForDN(dn, usersList=None): if not result["OK"]: return result usersList = result["Value"] - for username in usersList: - if dn in gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/DN", []): - return S_OK(username) - return S_ERROR("No username found for dn %s" % dn) + for userName in usersList: + if dn in gConfig.getValue(f"{gBaseRegistrySection}/Users/{userName}/DN", []): + return S_OK(userName) + return S_ERROR("No user name found for dn %s" % dn) -def getDNForUsername(username): +def getDNForUsername(userName): """Get user DN for user - :param str username: user name + :param str userName: user name :return: S_OK(str)/S_ERROR() """ - dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/DN", []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) + dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{userName}/DN", []) + return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % userName) def getDNForHost(host): @@ -89,14 +89,14 @@ def __getGroupsWithAttr(attrName, value): return S_OK(groups) if groups else S_ERROR(f"No groups found for {attrName}={value}") -def getGroupsForUser(username): +def getGroupsForUser(userName): """Find groups for user - :param str username: user name + :param str userName: user name :return: S_OK(list)/S_ERROR() -- contain list of groups """ - return __getGroupsWithAttr("Users", username) + return __getGroupsWithAttr("Users", userName) def getGroupsForVO(vo): @@ -430,7 +430,7 @@ def getIdPForGroup(group): :return: str """ - return getVOOption(getVOForGroup(group), "IdP") + return getVOOption(getVOForGroup(group), "IdProvider") def getDefaultVOMSAttribute(): @@ -590,21 +590,21 @@ def getUsernameForID(ID, usersList=None): if not result["OK"]: return result usersList = result["Value"] - for username in usersList: - if ID in gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/ID", []): - return S_OK(username) - return S_ERROR("No username found for ID %s" % ID) + for userName in usersList: + if ID in gConfig.getValue(f"{gBaseRegistrySection}/Users/{userName}/ID", []): + return S_OK(userName) + return S_ERROR("No user name found for ID %s" % ID) -def getCAForUsername(username): +def getCAForUsername(userName): """Get CA option by user name - :param str username: user name + :param str userName: user name :return: S_OK(str)/S_ERROR() """ - dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/CA", []) - return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % username) + dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{userName}/CA", []) + return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % userName) def getDNProperty(userDN, value, defaultValue=None): @@ -675,15 +675,15 @@ def isDownloadableGroup(groupName): return True -def getUserDict(username): +def getUserDict(userName): """Get full information from user section - :param str username: DIRAC user name + :param str userName: DIRAC user name :return: S_OK()/S_ERROR() """ resDict = {} - relPath = f"{gBaseRegistrySection}/Users/{username}/" + relPath = f"{gBaseRegistrySection}/Users/{userName}/" result = gConfig.getConfigurationTree(relPath) if not result["OK"]: return result @@ -701,8 +701,8 @@ def getEmailsForGroup(groupName): :return: list(list) -- inner list contains emails for a user """ emails = [] - for username in getUsersInGroup(groupName): - email = getUserOption(username, "Email", []) + for userName in getUsersInGroup(groupName): + email = getUserOption(userName, "Email", []) emails.append(email) return emails diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py index a85ac81e6eb..ec5b62634f8 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py @@ -84,7 +84,6 @@ def getGOCSiteName(diracSiteName): def getGOCSites(diracSites=None): - if diracSites is None: diracSites = getSites() if not diracSites["OK"]: @@ -229,8 +228,17 @@ def getQueue(site, ce, queue): return S_OK(resultDict) -def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None): - """Get CE/queue options according to the specified selection""" +def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, tags=None): + """Get CE/queue options according to the specified selection + + :param str list siteList: sites to be selected + :param str list ceList: CEs to be selected + :param str list ceTypeList: CE types to be selected + :param str community: selected VO + :param str list tags: tags required for selection + + :return: S_OK/S_ERROR with Value - dictionary of selected Site/CE/Queue parameters + """ result = gConfig.getSections("/Resources/Sites") if not result["OK"]: @@ -260,6 +268,7 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None): continue ces = result["Value"] for ce in ces: + ceTags = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Tag", []) if ceTypeList: ceType = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/CEType", "") if not ceType or ceType not in ceTypeList: @@ -284,6 +293,11 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None): comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/VO", []) if comList and community.lower() not in [cl.lower() for cl in comList]: continue + if tags: + queueTags = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/Tag", []) + queueTags = set(ceTags + queueTags) + if not queueTags or not set(tags).issubset(queueTags): + continue resultDict.setdefault(site, {}) resultDict[site].setdefault(ce, ceOptionsDict) resultDict[site][ce].setdefault("Queues", {}) diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py index 570eb4dc969..438790456a9 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py @@ -63,7 +63,6 @@ ], ) def test_getDIRACPlatform(mocker, mockGCReplyInput, requested, expectedRes, expectedValue): - mockGCReply.return_value = mockGCReplyInput mocker.patch("DIRAC.Interfaces.API.Dirac.gConfig.getOptionsDict", side_effect=mockGCReply) diff --git a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py index 8278295c185..ca75ecfc095 100755 --- a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py @@ -221,7 +221,6 @@ def getServiceURLs(system, service=None, setup=False, failover=False): # Be sure that urls not None for url in urls or []: - # Trying if we are refering to the list of main servers # which would be like dips://$MAINSERVERS$:1234/System/Component if "$MAINSERVERS$" in url: diff --git a/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py b/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py index d4d0c620fc2..2ce15174696 100644 --- a/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py +++ b/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py @@ -66,7 +66,6 @@ def __refreshLoop(self): for official documentation about this type of method. """ while self._automaticUpdate: - # This is the sleep from Tornado, like a sleep it wait some time # But this version is non-blocking, so IOLoop can continue execution yield gen.sleep(gConfigurationData.getPropagationTime()) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py index 8cfa74d79c7..c470cdefc46 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py @@ -20,7 +20,6 @@ def processScriptSwitches(): - global vo, dry, doCEs, hostURL, onecore Script.registerSwitch("V:", "vo=", "Virtual Organization") @@ -55,7 +54,6 @@ def processScriptSwitches(): def checkUnusedCEs(): - global vo, dry, ceBdiiDict, hostURL gLogger.notice("looking for new computing resources in the BDII database...") @@ -207,7 +205,6 @@ def checkUnusedCEs(): def updateCS(changeSet): - global vo, dry, ceBdiiDict changeList = sorted(changeSet) @@ -241,7 +238,6 @@ def updateCS(changeSet): def updateSites(): - global vo, dry, ceBdiiDict, onecore result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, onecore=onecore) diff --git a/src/DIRAC/Core/Base/AgentModule.py b/src/DIRAC/Core/Base/AgentModule.py index e36aebc8a5c..5aa36020c2b 100644 --- a/src/DIRAC/Core/Base/AgentModule.py +++ b/src/DIRAC/Core/Base/AgentModule.py @@ -147,7 +147,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}): self.activityMonitoring = True def __getCodeInfo(self): - try: self.__codeProperties["version"] = importlib.metadata.version( inspect.getmodule(self).__package__.split(".")[0] diff --git a/src/DIRAC/Core/Base/CLI.py b/src/DIRAC/Core/Base/CLI.py index 9361b30d807..9a878212720 100644 --- a/src/DIRAC/Core/Base/CLI.py +++ b/src/DIRAC/Core/Base/CLI.py @@ -38,13 +38,11 @@ def colorize(text, color): class CLI(cmd.Cmd): def __init__(self): - cmd.Cmd.__init__(self) self.indentSpace = 20 self._initSignals() def _handleSignal(self, sig, frame): - print("\nReceived signal", sig, ", exiting ...") self.do_quit(self) diff --git a/src/DIRAC/Core/Base/DB.py b/src/DIRAC/Core/Base/DB.py index 55a91f592c2..6b386aa1580 100755 --- a/src/DIRAC/Core/Base/DB.py +++ b/src/DIRAC/Core/Base/DB.py @@ -10,7 +10,6 @@ class DB(DIRACDB, MySQL): """All DIRAC DB classes should inherit from this one (unless using sqlalchemy)""" def __init__(self, dbname, fullname, debug=False, parentLogger=None): - self.fullname = fullname result = getDBParameters(fullname) diff --git a/src/DIRAC/Core/Base/ExecutorMindHandler.py b/src/DIRAC/Core/Base/ExecutorMindHandler.py index 90369afbb35..69eff56cf54 100644 --- a/src/DIRAC/Core/Base/ExecutorMindHandler.py +++ b/src/DIRAC/Core/Base/ExecutorMindHandler.py @@ -10,7 +10,6 @@ class ExecutorMindHandler(RequestHandler): - MSG_DEFINITIONS = { "ProcessTask": {"taskId": int, "taskStub": str, "eType": str}, "TaskDone": {"taskId": int, "taskStub": str}, diff --git a/src/DIRAC/Core/DISET/ServiceReactor.py b/src/DIRAC/Core/DISET/ServiceReactor.py index 76aead02d43..f8be2fb486b 100644 --- a/src/DIRAC/Core/DISET/ServiceReactor.py +++ b/src/DIRAC/Core/DISET/ServiceReactor.py @@ -42,7 +42,6 @@ class ServiceReactor: - __transportExtraKeywords = { "SSLSessionTimeout": False, "IgnoreCRLs": False, diff --git a/src/DIRAC/Core/DISET/private/FileHelper.py b/src/DIRAC/Core/DISET/private/FileHelper.py index e5febfb6fb0..4ce3b224ad0 100755 --- a/src/DIRAC/Core/DISET/private/FileHelper.py +++ b/src/DIRAC/Core/DISET/private/FileHelper.py @@ -14,7 +14,6 @@ class FileHelper: - __validDirections = ("toClient", "fromClient", "receive", "send") __directionsMapping = {"toClient": "send", "fromClient": "receive"} diff --git a/src/DIRAC/Core/DISET/private/MessageFactory.py b/src/DIRAC/Core/DISET/private/MessageFactory.py index dafddbbd5e9..6304ea5b938 100644 --- a/src/DIRAC/Core/DISET/private/MessageFactory.py +++ b/src/DIRAC/Core/DISET/private/MessageFactory.py @@ -93,7 +93,6 @@ def __loadMessagesForAncestry(self, handlerClass): class Message: - DEFAULTWAITFORACK = False def __init__(self, msgName, msgDefDict): diff --git a/src/DIRAC/Core/DISET/private/Service.py b/src/DIRAC/Core/DISET/private/Service.py index c22cbd74f32..ef54ad93fcf 100644 --- a/src/DIRAC/Core/DISET/private/Service.py +++ b/src/DIRAC/Core/DISET/private/Service.py @@ -33,7 +33,6 @@ class Service: - SVC_VALID_ACTIONS = {"RPC": "export", "FileTransfer": "transfer", "Message": "msg", "Connection": "Message"} SVC_SECLOG_CLIENT = SecurityLogClient() @@ -182,7 +181,6 @@ def _loadHandlerInit(self): return S_OK(handlerInfo) def _loadActions(self): - handlerClass = self._handler["class"] authRules = {} @@ -520,7 +518,6 @@ def _processProposal(self, trid, proposalTuple, handlerObj): messageConnection = True if messageConnection: - if self._msgBroker.getNumConnections() > self._cfg.getMaxMessagingConnections(): result = S_ERROR("Maximum number of connections reached. Try later") result["closeTransport"] = True diff --git a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py index 551672ce0bc..d58e20cd74d 100755 --- a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py @@ -119,8 +119,7 @@ def initAsClient(self): # We ignore the returned sockaddr because SSL.Connection.connect needs # a host name. addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - for (family, _socketType, _proto, _canonname, _socketAddress) in addrInfoList: - + for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList: try: self.oSocket = SSL.Connection(self.__ctx, family=family) diff --git a/src/DIRAC/Core/DISET/test/Test_AuthManager.py b/src/DIRAC/Core/DISET/test/Test_AuthManager.py index 1e1b969981e..e509dc5e483 100644 --- a/src/DIRAC/Core/DISET/test/Test_AuthManager.py +++ b/src/DIRAC/Core/DISET/test/Test_AuthManager.py @@ -128,7 +128,6 @@ def tearDown(self): pass def test_userProperties(self): - # MethodAll accepts everybody result = self.authMgr.authQuery("MethodAll", self.userCredDict) self.assertTrue(result) @@ -166,7 +165,6 @@ def test_userProperties(self): self.assertTrue(result) def test_userGroup(self): - # MethodAllGroup accepts everybody from the right group result = self.authMgr.authQuery("MethodAllGroup", self.userCredDict) self.assertTrue(result) @@ -240,7 +238,6 @@ def test_userGroup(self): self.assertTrue(result) def test_userVO(self): - # MethodAllGroup accepts everybody from the right group result = self.authMgr.authQuery("MethodAllVO", self.userCredDict) self.assertTrue(result) @@ -278,7 +275,6 @@ def test_userVO(self): self.assertFalse(result) def test_hostProperties(self): - # MethodAll accepts everybody result = self.authMgr.authQuery("MethodAll", self.hostCredDict) self.assertTrue(result) diff --git a/src/DIRAC/Core/LCG/GGUSTicketsClient.py b/src/DIRAC/Core/LCG/GGUSTicketsClient.py index e7bf839fb9d..98592c5a162 100644 --- a/src/DIRAC/Core/LCG/GGUSTicketsClient.py +++ b/src/DIRAC/Core/LCG/GGUSTicketsClient.py @@ -30,7 +30,6 @@ def getGGUSURL(vo=None, siteName=None): class GGUSTicketsClient: def __init__(self): - # create client instance using GGUS wsdl: self.gclient = Client("https://prod-ars.ggus.eu/arsys/WSDL/public/prod-ars/GGUS") authInfo = self.gclient.factory.create("AuthenticationInfo") @@ -88,7 +87,6 @@ def globalStatistics(self, ticketList): terminalStates = ["solved", "unsolved", "verified", "closed"] for ticket in ticketList: - _id = str(ticket.GHD_Request_ID) _status = str(ticket.GHD_Status) _shortDescription = str(ticket.GHD_Short_Description) diff --git a/src/DIRAC/Core/LCG/GOCDBClient.py b/src/DIRAC/Core/LCG/GOCDBClient.py index 608d94b592e..624e1ad08ec 100644 --- a/src/DIRAC/Core/LCG/GOCDBClient.py +++ b/src/DIRAC/Core/LCG/GOCDBClient.py @@ -198,7 +198,6 @@ def getCurrentDTLinkList(self): ############################################################################# def getHostnameDowntime(self, hostname, startDate=None, ongoing=False): - params = hostname if startDate and ongoing: diff --git a/src/DIRAC/Core/LCG/test/Test_LCG.py b/src/DIRAC/Core/LCG/test/Test_LCG.py index af7f3594f45..4ab3cf9aac5 100644 --- a/src/DIRAC/Core/LCG/test/Test_LCG.py +++ b/src/DIRAC/Core/LCG/test/Test_LCG.py @@ -330,7 +330,6 @@ def test__downTimeXMLParsing_affected(): - res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing, "Resource") assert set(res) == {"109962G0 lhcbsrm-kit.gridka.deSRM"} assert res["109962G0 lhcbsrm-kit.gridka.deSRM"]["HOSTNAME"] == "lhcbsrm-kit.gridka.de" @@ -366,7 +365,6 @@ def test__downTimeXMLParsing_affected(): def test__downTimeXMLParsing(): - res = GOCCli._downTimeXMLParsing(XML_site_ongoing, "Site") assert set(res) == {"28490G0 GRISU-ENEA-GRID"} assert res["28490G0 GRISU-ENEA-GRID"]["SITENAME"] == "GRISU-ENEA-GRID" diff --git a/src/DIRAC/Core/Security/m2crypto/X509Chain.py b/src/DIRAC/Core/Security/m2crypto/X509Chain.py index cd1c7bc9a29..e938ba32b72 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509Chain.py +++ b/src/DIRAC/Core/Security/m2crypto/X509Chain.py @@ -612,7 +612,6 @@ def __checkProxyness(self): # Here we make sure that each certificate in the chain was # signed by the previous one for step in range(len(self._certList) - 1): - # this is a cryptographic check with the keys issuerMatch = self.__checkIssuer(step, step + 1) if not issuerMatch: diff --git a/src/DIRAC/Core/Security/test/Test_X509Chain.py b/src/DIRAC/Core/Security/test/Test_X509Chain.py index 40189c818f7..d5b97bdeb6c 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Chain.py +++ b/src/DIRAC/Core/Security/test/Test_X509Chain.py @@ -528,6 +528,7 @@ def test_getIssuerCert(get_proxy): # From now on, test proxy coming from Requests ################################################################ + # # retVal = chain.generateChainFromRequestString(reqDict['request'], # lifetime=chainLifeTime, diff --git a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py index 815fa227bd5..c6b622d70d4 100644 --- a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py @@ -128,7 +128,6 @@ def __init__(self, serviceName, **kwargs): self.__discoverExtraCredentials, self.__discoverURL, ): - result = initFunc() if not result["OK"] and self.__initStatus["OK"]: self.__initStatus = result diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index 3e5c364cb4b..4cf625c4325 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -14,7 +14,6 @@ from functools import partial import jwt -import tornado from tornado.web import RequestHandler, HTTPError from tornado.ioloop import IOLoop @@ -445,7 +444,6 @@ def __initialize(cls, request): # Otherwise, do the work but with a lock with cls.__init_lock: - # Check again that the initialization was not done by another thread # while we were waiting for the lock if cls.__init_done: diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py index a12e13887e2..78b1bea97b1 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py @@ -13,7 +13,6 @@ @Script() def main(): - if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py index f71d381c0c1..b005bd4055c 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py @@ -13,7 +13,6 @@ @Script() def main(): - if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." diff --git a/src/DIRAC/Core/Utilities/DEncode.py b/src/DIRAC/Core/Utilities/DEncode.py index 777aa0a7d43..29ee001219a 100755 --- a/src/DIRAC/Core/Utilities/DEncode.py +++ b/src/DIRAC/Core/Utilities/DEncode.py @@ -493,7 +493,6 @@ def decodeDict(data, i): oD = {} i += 1 while data[i] != _ord("e"): - if DIRAC_DEBUG_DENCODE_CALLSTACK: # If we have numbers as keys if data[i] in (_ord("i"), _ord("I"), _ord("f")): diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py index fad6a0b3893..ab93e240ec0 100644 --- a/src/DIRAC/Core/Utilities/DIRACScript.py +++ b/src/DIRAC/Core/Utilities/DIRACScript.py @@ -1,6 +1,7 @@ from DIRAC.Core.Base.Script import Script from DIRAC.Core.Utilities.Decorators import deprecated + # TODO: remove it in 8.1 @deprecated("DIRACScript is deprecated, use 'from DIRAC.Core.Base.Script import Script' instead.") class DIRACScript(Script): diff --git a/src/DIRAC/Core/Utilities/ElasticSearchDB.py b/src/DIRAC/Core/Utilities/ElasticSearchDB.py index c41960d6600..5c6219c2965 100644 --- a/src/DIRAC/Core/Utilities/ElasticSearchDB.py +++ b/src/DIRAC/Core/Utilities/ElasticSearchDB.py @@ -57,7 +57,6 @@ def generateDocs(data, withTimeStamp=True): :return: doc """ for doc in copy.deepcopy(data): - if withTimeStamp: if "timestamp" not in doc: sLog.warn("timestamp is not given") diff --git a/src/DIRAC/Core/Utilities/Graphs/BarGraph.py b/src/DIRAC/Core/Utilities/Graphs/BarGraph.py index ca0003873cf..3b9ca13b99c 100644 --- a/src/DIRAC/Core/Utilities/Graphs/BarGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/BarGraph.py @@ -29,7 +29,6 @@ class BarGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) if "span" in self.prefs: self.width = self.prefs["span"] @@ -41,7 +40,6 @@ def __init__(self, data, ax, prefs, *args, **kw): self.width = (max(self.gdata.all_keys) - min(self.gdata.all_keys)) / (nKeys - 1) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -181,7 +179,6 @@ def x_formatter_cb(self, ax): xmin = 0 ax.set_xlim(xmin=xmin, xmax=len(ticks)) elif self.gdata.key_type == "time": - # ax.set_xlim( xmin=self.begin_num,xmax=self.end_num ) dl = PrettyDateLocator() df = PrettyDateFormatter(dl) diff --git a/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py b/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py index 07a57a34e08..a8f60fdf74a 100644 --- a/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py @@ -23,11 +23,9 @@ class CurveGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) diff --git a/src/DIRAC/Core/Utilities/Graphs/Graph.py b/src/DIRAC/Core/Utilities/Graphs/Graph.py index 8e958fe55fe..1e87bc0d0e7 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Graph.py +++ b/src/DIRAC/Core/Utilities/Graphs/Graph.py @@ -24,7 +24,6 @@ def __init__(self, *args, **kw): super().__init__(*args, **kw) def layoutFigure(self, legend): - prefs = self.prefs # Get the main Figure object @@ -182,7 +181,6 @@ def makeTextGraph(self, text="Empty image"): figure.text(0.5, 0.5, text, horizontalalignment="center", size=pixelToPoint(text_size, dpi)) def makeGraph(self, data, *args, **kw): - start = time.time() # Evaluate all the preferences diff --git a/src/DIRAC/Core/Utilities/Graphs/GraphData.py b/src/DIRAC/Core/Utilities/Graphs/GraphData.py index 07aecd620c1..8c226ee7888 100644 --- a/src/DIRAC/Core/Utilities/Graphs/GraphData.py +++ b/src/DIRAC/Core/Utilities/Graphs/GraphData.py @@ -53,7 +53,6 @@ def get_key_type(keys): class GraphData: def __init__(self, data={}): - self.truncated = 0 self.all_keys = [] self.labels = [] @@ -76,7 +75,6 @@ def setData(self, data): self.initialize() def initialize(self, key_type=None): - keys = list(self.data) if not keys: print("GraphData Error: empty data") @@ -110,13 +108,11 @@ def initialize(self, key_type=None): self.sortLabels() def expandKeys(self): - if not self.plotdata: for sub in self.subplots: self.subplots[sub].expandKeys(self.all_keys) def isSimplePlot(self): - return self.plotdata is not None def sortLabels(self, sort_type="max_value", reverse_order=False): @@ -247,11 +243,9 @@ def getStringMap(self): return self.all_string_map def getNumberOfKeys(self): - return len(self.all_keys) def getNumberOfLabels(self): - if self.truncated: return self.truncated + 1 else: @@ -364,7 +358,6 @@ class PlotData: """PlotData class is a container for a one dimensional plot data""" def __init__(self, data, single=True, key_type=None): - self.key_type = "unknown" if not data: print("PlotData Error: empty data") @@ -390,7 +383,6 @@ def __init__(self, data, single=True, key_type=None): self.initialize() def initialize(self): - if self.key_type == "string": self.keys = self.sortKeys("weight") else: @@ -540,7 +532,6 @@ def parseData(self, key_type=None): self.keys = list(self.parsed_data) def makeCumulativePlot(self): - if not self.sorted_keys: self.sortKeys() @@ -559,19 +550,15 @@ def makeCumulativePlot(self): self.last_value = float(self.values[-1]) def getPlotData(self): - return self.parsed_data def getPlotErrors(self): - return self.parsed_errors def getPlotNumData(self): - return zip(self.num_keys, self.values, self.errors) def getPlotDataForNumKeys(self, num_keys, zeroes=False): - result_pairs = [] for num_key in num_keys: try: @@ -589,21 +576,16 @@ def getPlotDataForNumKeys(self, num_keys, zeroes=False): return result_pairs def getKeys(self): - return self.keys def getNumKeys(self): - return self.num_keys def getValues(self): - return self.values def getMaxValue(self): - return max(self.values) def getMinValue(self): - return min(self.values) diff --git a/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py b/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py index dffec7ee8e4..bcc1cf7cd58 100644 --- a/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py +++ b/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py @@ -100,7 +100,6 @@ def convert_to_datetime(dstring): def to_timestamp(val): - try: v = float(val) if v > 1000000000 and v < 1900000000: @@ -373,7 +372,6 @@ def get_locator(self, dmin, dmax): def pretty_float(num): - if num > 1000: return comma_format(int(num)) @@ -465,7 +463,6 @@ def makeDataFromCSV(csv): def darkenColor(color, factor=2): - c1 = int(color[1:3], 16) c2 = int(color[3:5], 16) c3 = int(color[5:7], 16) diff --git a/src/DIRAC/Core/Utilities/Graphs/Legend.py b/src/DIRAC/Core/Utilities/Graphs/Legend.py index fc5601c4f4c..69e6832c5f0 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Legend.py +++ b/src/DIRAC/Core/Utilities/Graphs/Legend.py @@ -15,7 +15,6 @@ class Legend: def __init__(self, data=None, axes=None, *aw, **kw): - self.text_size = 0 self.column_width = 0 self.labels = {} @@ -45,16 +44,13 @@ def __init__(self, data=None, axes=None, *aw, **kw): self.__get_column_width() def dumpPrefs(self): - for key in self.prefs: print(key.rjust(20), ":", str(self.prefs[key]).ljust(40)) def setLabels(self, labels): - self.labels = labels def setAxes(self, axes): - self.ax = axes self.canvas = self.ax.figure.canvas self.ax.set_axis_off() @@ -86,7 +82,6 @@ def getLegendSize(self): return legend_width, legend_height, legend_max_height def __get_legend_text_size(self): - text_size = self.prefs["text_size"] text_padding = self.prefs["text_padding"] legend_text_size = self.prefs.get("legend_text_size", text_size) @@ -94,7 +89,6 @@ def __get_legend_text_size(self): return legend_text_size, legend_text_padding def __get_column_width(self): - max_length = 0 max_column_text = "" flag = self.prefs.get("legend_numbers", True) @@ -135,7 +129,6 @@ def __get_column_width(self): ) def draw(self): - dpi = self.prefs["dpi"] ax_xsize = self.ax.get_window_extent().width ax_ysize = self.ax.get_window_extent().height diff --git a/src/DIRAC/Core/Utilities/Graphs/LineGraph.py b/src/DIRAC/Core/Utilities/Graphs/LineGraph.py index f7c7e1d55ed..35a5e06987b 100644 --- a/src/DIRAC/Core/Utilities/Graphs/LineGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/LineGraph.py @@ -25,11 +25,9 @@ class LineGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -71,7 +69,6 @@ def draw(self): labels = [(color, 0.0)] for label, num in labels: - color = self.palette.getColor(label) ind = 0 tmp_x = [] diff --git a/src/DIRAC/Core/Utilities/Graphs/Palette.py b/src/DIRAC/Core/Utilities/Graphs/Palette.py index 460d039f929..1f4bc001f71 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Palette.py +++ b/src/DIRAC/Core/Utilities/Graphs/Palette.py @@ -64,7 +64,6 @@ class Palette: def __init__(self, palette={}, colors=[]): - self.palette = country_palette self.palette.update(job_status_palette) self.palette.update(miscelaneous_pallette) @@ -80,14 +79,12 @@ def addPalette(self, palette): self.palette.update(palette) def getColor(self, label): - if label in self.palette: return self.palette[label] else: return self.generateColor(label) def generateColor(self, label): - myMD5 = hashlib.md5() myMD5.update(label.encode()) hexstring = myMD5.hexdigest() diff --git a/src/DIRAC/Core/Utilities/Graphs/PieGraph.py b/src/DIRAC/Core/Utilities/Graphs/PieGraph.py index 420c733bbb7..91c887e53eb 100644 --- a/src/DIRAC/Core/Utilities/Graphs/PieGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/PieGraph.py @@ -14,12 +14,10 @@ class PieGraph(PlotBase): def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) self.pdata = data def pie(self, explode=None, colors=None, autopct=None, pctdistance=0.6, shadow=False): - start = time.time() labels = self.pdata.getLabels() if labels[0][0] == "NoLabels": @@ -155,11 +153,9 @@ def pie(self, explode=None, colors=None, autopct=None, pctdistance=0.6, shadow=F min_amount = 0.1 def getLegendData(self): - return self.legendData def draw(self): - self.ylabel = "" self.prefs["square_axis"] = True PlotBase.draw(self) diff --git a/src/DIRAC/Core/Utilities/Graphs/PlotBase.py b/src/DIRAC/Core/Utilities/Graphs/PlotBase.py index 135874ff1c7..de7fb719bdc 100644 --- a/src/DIRAC/Core/Utilities/Graphs/PlotBase.py +++ b/src/DIRAC/Core/Utilities/Graphs/PlotBase.py @@ -16,7 +16,6 @@ class PlotBase: def __init__(self, data=None, axes=None, *aw, **kw): - self.ax_contain = axes self.canvas = None self.figure = None @@ -34,12 +33,10 @@ def __init__(self, data=None, axes=None, *aw, **kw): self.gdata = data def dumpPrefs(self): - for key in self.prefs: print(key.rjust(20), ":", str(self.prefs[key]).ljust(40)) def setAxes(self, axes): - self.ax_contain = axes self.ax_contain.set_axis_off() self.figure = self.ax_contain.get_figure() @@ -47,7 +44,6 @@ def setAxes(self, axes): self.dpi = self.ax_contain.figure.get_dpi() def draw(self): - prefs = self.prefs dpi = self.ax_contain.figure.get_dpi() diff --git a/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py b/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py index c0a8fb81774..39c082801d2 100644 --- a/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py @@ -47,7 +47,6 @@ class QualityMapGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) if isinstance(data, dict): self.gdata = GraphData(data) @@ -99,7 +98,6 @@ def get_alpha(*args, **kw): self.mapper = mapper def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -134,7 +132,6 @@ def draw(self): for label, _num in labels: labelNames.append(label) for key, value, _error in self.gdata.getPlotNumData(label): - if xmin is None or xmin > (key + offset): xmin = key + offset if xmax is None or xmax < (key + offset): @@ -183,13 +180,11 @@ def draw(self): # setp( cb.ax.get_xticklabels(), fontname = self.prefs['font'] ) def getQualityColor(self, value): - if value is None or value < 0.0: return "#FFFFFF" return self.mapper.to_rgba(value) def getLegendData(self): - return None def x_formatter_cb(self, ax): @@ -209,7 +204,6 @@ def x_formatter_cb(self, ax): xmin = 0.0 ax.set_xlim(xmin=xmin, xmax=len(ticks)) elif self.gdata.key_type == "time": - # ax.set_xlim( xmin=self.begin_num,xmax=self.end_num ) dl = PrettyDateLocator() df = PrettyDateFormatter(dl) diff --git a/src/DIRAC/Core/Utilities/Graphs/__init__.py b/src/DIRAC/Core/Utilities/Graphs/__init__.py index d8bf696fabe..0b721155c08 100644 --- a/src/DIRAC/Core/Utilities/Graphs/__init__.py +++ b/src/DIRAC/Core/Utilities/Graphs/__init__.py @@ -113,7 +113,6 @@ def graph(data, fileName, *args, **kw): - prefs = evalPrefs(*args, **kw) graph_size = prefs.get("graph_size", "normal") diff --git a/src/DIRAC/Core/Utilities/Grid.py b/src/DIRAC/Core/Utilities/Grid.py index fbd6e9066b2..155d0da5a4d 100644 --- a/src/DIRAC/Core/Utilities/Grid.py +++ b/src/DIRAC/Core/Utilities/Grid.py @@ -11,7 +11,7 @@ from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall -def executeGridCommand(proxy, cmd, gridEnvScript=None): +def executeGridCommand(proxy, cmd, gridEnvScript=None, gridEnvDict=None): """ Execute cmd tuple after sourcing GridEnv """ @@ -53,6 +53,9 @@ def executeGridCommand(proxy, cmd, gridEnvScript=None): return ret gridEnv["X509_USER_PROXY"] = ret["Value"] + if gridEnvDict: + gridEnv.update(gridEnvDict) + result = systemCall(120, cmd, env=gridEnv) return result diff --git a/src/DIRAC/Core/Utilities/ModuleFactory.py b/src/DIRAC/Core/Utilities/ModuleFactory.py index 623708a5def..7031854dd18 100755 --- a/src/DIRAC/Core/Utilities/ModuleFactory.py +++ b/src/DIRAC/Core/Utilities/ModuleFactory.py @@ -10,7 +10,6 @@ class ModuleFactory: - ############################################################################# def __init__(self): """Standard constructor""" diff --git a/src/DIRAC/Core/Utilities/MySQL.py b/src/DIRAC/Core/Utilities/MySQL.py index fe87d4df661..e204b5d5027 100755 --- a/src/DIRAC/Core/Utilities/MySQL.py +++ b/src/DIRAC/Core/Utilities/MySQL.py @@ -249,7 +249,6 @@ def captureOptimizerTraces(meth): @functools.wraps(meth) def innerMethod(self, *args, **kwargs): - # First, get a connection to the DB, and enable the tracing connection = self._MySQL__connectionPool.get(self._MySQL__dbName)["Value"] connection.cursor().execute('SET optimizer_trace="enabled=on";') @@ -551,7 +550,6 @@ def _except(self, methodName, x, err, cmd="", print=True): return S_ERROR(DErrno.EMYSQL, f"{err}: ({repr(e)})") def __isDateTime(self, dateString): - if dateString == "UTC_TIMESTAMP()": return True try: @@ -840,7 +838,6 @@ def _createViews(self, viewsDict, force=False): # gLogger.debug(viewsDict) for viewName, viewDict in viewsDict.items(): - viewQuery = [f"CREATE OR REPLACE VIEW `{self.__dbName}`.`{viewName}` AS"] columns = ",".join([f"{colDef} AS {colName}" for colName, colDef in viewDict.get("Fields", {}).items()]) @@ -1004,7 +1001,6 @@ def _createTables(self, tableDict, force=False): if "ForeignKeys" in thisTable: thisKeys = thisTable["ForeignKeys"] for key, auxTable in thisKeys.items(): - forTable = auxTable.split(".")[0] forKey = key if forTable != auxTable: diff --git a/src/DIRAC/Core/Utilities/Pfn.py b/src/DIRAC/Core/Utilities/Pfn.py index 6c05be2f8c7..5bbee3e7237 100755 --- a/src/DIRAC/Core/Utilities/Pfn.py +++ b/src/DIRAC/Core/Utilities/Pfn.py @@ -211,7 +211,6 @@ def default_pfnparse(pfn): return S_ERROR("wrong 'pfn' argument value in function call, expected non-empty string, got %s" % str(pfn)) pfnDict = dict.fromkeys(["Protocol", "Host", "Port", "WSUrl", "Path", "FileName"], "") try: - parsed = parse.urlparse(pfn) pfnDict["Protocol"] = parsed.scheme if ":" in parsed.netloc: diff --git a/src/DIRAC/Core/Utilities/PrettyPrint.py b/src/DIRAC/Core/Utilities/PrettyPrint.py index 2ca0cae1070..898512fd198 100644 --- a/src/DIRAC/Core/Utilities/PrettyPrint.py +++ b/src/DIRAC/Core/Utilities/PrettyPrint.py @@ -35,7 +35,6 @@ def printTable(fields, records, sortField="", numbering=True, printOut=True, col """ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): - justification = None if isinstance(value, dict): justification = value.get("Just") diff --git a/src/DIRAC/Core/Utilities/ProcessPool.py b/src/DIRAC/Core/Utilities/ProcessPool.py index b39833a80da..5384431e71a 100644 --- a/src/DIRAC/Core/Utilities/ProcessPool.py +++ b/src/DIRAC/Core/Utilities/ProcessPool.py @@ -256,7 +256,6 @@ def run(self): # main loop while True: - # draining, stopEvent is set, exiting if self.__stopEvent.is_set(): return diff --git a/src/DIRAC/Core/Utilities/Proxy.py b/src/DIRAC/Core/Utilities/Proxy.py index 46e7b600d7f..62c4e87a411 100644 --- a/src/DIRAC/Core/Utilities/Proxy.py +++ b/src/DIRAC/Core/Utilities/Proxy.py @@ -62,7 +62,6 @@ def executeWithUserProxy(fcn): """ def wrapped_fcn(*args, **kwargs): - userName = kwargs.pop("proxyUserName", "") userDN = kwargs.pop("proxyUserDN", "") userGroup = kwargs.pop("proxyUserGroup", "") @@ -71,7 +70,6 @@ def wrapped_fcn(*args, **kwargs): executionLockFlag = kwargs.pop("executionLock", False) if (userName or userDN) and userGroup: - proxyResults = _putProxy( userName=userName, userDN=userDN, @@ -149,7 +147,6 @@ def executeWithoutServerCertificate(fcn): """ def wrapped_fcn(*args, **kwargs): - # Get the lock and acquire it executionLock = LockRing().getLock("_UseUserProxy_", recursive=True) executionLock.acquire() diff --git a/src/DIRAC/Core/Utilities/SiteSEMapping.py b/src/DIRAC/Core/Utilities/SiteSEMapping.py index fea6a136952..5c3883f1de0 100644 --- a/src/DIRAC/Core/Utilities/SiteSEMapping.py +++ b/src/DIRAC/Core/Utilities/SiteSEMapping.py @@ -64,7 +64,6 @@ def getStorageElementsHosts(seNames=None): seNames = DMSHelpers().getStorageElements() for seName in seNames: - try: seHost = getSEHosts(seName) if not seHost["OK"]: diff --git a/src/DIRAC/Core/Utilities/test/Test_List.py b/src/DIRAC/Core/Utilities/test/Test_List.py index 361976202af..05031ea9257 100644 --- a/src/DIRAC/Core/Utilities/test/Test_List.py +++ b/src/DIRAC/Core/Utilities/test/Test_List.py @@ -13,6 +13,7 @@ # sut from DIRAC.Core.Utilities import List + ######################################################################## class ListTestCase(unittest.TestCase): """py:class ListTestCase diff --git a/src/DIRAC/Core/Utilities/test/Test_Time.py b/src/DIRAC/Core/Utilities/test/Test_Time.py index 1ee9494d655..e4885e63183 100755 --- a/src/DIRAC/Core/Utilities/test/Test_Time.py +++ b/src/DIRAC/Core/Utilities/test/Test_Time.py @@ -59,7 +59,6 @@ def tearDown(self): class TimeSuccess(TimeTestCase): def test_timeThis(self): - self.assertIsNone(myMethod()) self.assertIsNone(myClass().myMethodInAClass()) self.assertIsNone(myBetterClass().myMethodInAClass()) diff --git a/src/DIRAC/Core/Workflow/Parameter.py b/src/DIRAC/Core/Workflow/Parameter.py index fcbec1a2f50..053c963e032 100755 --- a/src/DIRAC/Core/Workflow/Parameter.py +++ b/src/DIRAC/Core/Workflow/Parameter.py @@ -580,7 +580,6 @@ def resolveGlobalVars(self, wf_parameters=None, step_parameters=None): substitute_vars = getSubstitute(v.value) while True: for substitute_var in substitute_vars: - # looking in the current scope v_other = self.find(substitute_var) diff --git a/src/DIRAC/Core/Workflow/Step.py b/src/DIRAC/Core/Workflow/Step.py index da545370c00..ba7910550b5 100755 --- a/src/DIRAC/Core/Workflow/Step.py +++ b/src/DIRAC/Core/Workflow/Step.py @@ -292,7 +292,6 @@ def execute(self, step_exec_attr, definitions): step_exec_attr[parameter.getLinkedParameter()], ) else: - setattr( step_exec_modules[mod_inst_name], parameter.getName(), diff --git a/src/DIRAC/Core/Workflow/Utility.py b/src/DIRAC/Core/Workflow/Utility.py index 83473738c6e..806310b41c2 100644 --- a/src/DIRAC/Core/Workflow/Utility.py +++ b/src/DIRAC/Core/Workflow/Utility.py @@ -48,11 +48,9 @@ def resolveVariables(varDict): def dataFromOption(parameter): - result = [] if parameter.type.lower() == "option": - fields = parameter.value.split(",") for f in fields: @@ -70,7 +68,6 @@ def dataFromOption(parameter): def expandDatafileOption(option): - result = "" if not re.search(";;", option.value): @@ -81,7 +78,6 @@ def expandDatafileOption(option): fname, ftype = files[0] fnames = fname.split(";;") if len(fnames) > 1: - template = option.value.strip().replace("=", "", 1) template = template.replace("{", "") template = template.replace("}", "") diff --git a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py index ac32b8822a4..a153d70069f 100644 --- a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py +++ b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py @@ -292,7 +292,6 @@ def monitorJobsLoop(self): log.debug("Getting active jobs") for loopId in range(nbOfLoops): - log.info("Getting next batch of jobs to monitor", f"{loopId}/{nbOfLoops}") # get jobs from DB res = self.fts3db.getActiveJobs(limit=JOB_MONITORING_BATCH_SIZE, jobAssignmentTag=self.assignmentTag) @@ -430,7 +429,6 @@ def _treatOperation(self, operation): ftsJob.cancel(context) if continueOperationProcessing: - res = operation.prepareNewJobs( maxFilesPerJob=self.maxFilesPerJob, maxAttemptsPerFile=self.maxAttemptsPerFile ) @@ -666,7 +664,6 @@ def endExecution(self): return self.dataOpSender.concludeSending() def __sendAccounting(self, ftsJob): - self.dataOpSender.sendData( ftsJob.accountingDict, commitFlag=True, diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py index 924cff187b3..f59d60af512 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py @@ -156,7 +156,6 @@ def __checkReplicas(self): return S_OK() def dmRemoval(self, toRemoveDict, targetSEs): - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(toRemoveDict))) self.rmsMonitoringReporter.commit() @@ -312,12 +311,10 @@ def dmTransfer(self, opFile): prString = f"file {lfn} replicated at {targetSE} in {repTime} s." if "register" in res["Value"]["Successful"][lfn]: - regTime = res["Value"]["Successful"][lfn]["register"] prString += " and registered in %s s." % regTime self.log.info(prString) else: - prString += " but failed to register" self.log.warn(prString) @@ -330,12 +327,10 @@ def dmTransfer(self, opFile): opFile.Error = "Failed to replicate" else: - reason = res["Value"]["Failed"][lfn] self.log.error("Failed to replicate and register", f"File {lfn} at {targetSE}: {reason}") opFile.Error = reason else: - opFile.Error = "DataManager error: %s" % res["Message"] self.log.error("DataManager error", res["Message"]) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py index 4160abb62fd..e5cb1045ab9 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py @@ -85,7 +85,6 @@ def __call__(self): removalStatus[lfn] = dict.fromkeys(targetSEs, "") for targetSE in targetSEs: - self.log.info("removing files from %s" % targetSE) # # 1st - bulk removal diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py index 531bbc3471d..8b5e65808ee 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py @@ -69,7 +69,6 @@ def __call__(self): self.log.error("Wrong value for TargetSE list, should contain only one target!", "%s" % targetSEs) self.operation.Error = "Wrong parameters: TargetSE should contain only one targetSE" for opFile in self.operation: - opFile.Status = "Failed" opFile.Error = "Wrong parameters: TargetSE should contain only one targetSE" @@ -138,9 +137,7 @@ def __call__(self): putAndRegister = putAndRegister["Successful"] if lfn in putAndRegister: - if "put" not in putAndRegister[lfn]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( lfn, "PutFail", targetSE, "", "PutAndRegister" ) @@ -152,7 +149,6 @@ def __call__(self): continue if "register" not in putAndRegister[lfn]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( lfn, "Put", targetSE, "", "PutAndRegister" ) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py index 2ef1c9a4d47..bb016d0ddd3 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py @@ -96,7 +96,6 @@ def __call__(self): se = StorageElement(targetSE) for opFile in toRetransfer.values(): - reTransfer = se.retransferOnlineFile(opFile.LFN) if not reTransfer["OK"]: opFile.Error = reTransfer["Message"] diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py index a1592f96361..a249adc7311 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py @@ -70,7 +70,6 @@ def __call__(self): # # loop over files for opFile in waitingFiles: - # # get LFN lfn = opFile.LFN # # and others @@ -85,7 +84,6 @@ def __call__(self): registerFile = dm.registerFile(fileTuple) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( @@ -107,7 +105,6 @@ def __call__(self): failedFiles += 1 else: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) # self.dataLoggingClient().addFileRecord( diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py index 53d3ffc3fd4..ce295dc230b 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py @@ -56,7 +56,6 @@ def __call__(self): registerOperations = {} successReplicas = 0 for opFile in waitingFiles: - # # get LFN lfn = opFile.LFN # # and others diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py index fef2be5e3dc..137a8fe46d7 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py @@ -180,7 +180,6 @@ def bulkRemoval(self, toRemoveDict): if lfn in bulkRemoval["Successful"]: opFile.Status = "Done" elif lfn in bulkRemoval["Failed"]: - error = bulkRemoval["Failed"][lfn] if isinstance(error, dict): error = ";".join([f"{k}-{v}" for k, v in error.items()]) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py index e2e8dc090bc..29ff285e555 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py @@ -89,7 +89,6 @@ def __call__(self): # # loop over targetSEs for targetSE in targetSEs: - self.log.info("Removing replicas at %s" % targetSE) # # 1st step - bulk removal diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py index a214a8001b4..191804532f0 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py @@ -585,28 +585,22 @@ def dmTransfer(self, fromFTS=False): catalogs = [cat.strip() for cat in catalogs.split(",")] for targetSE in self.operation.targetSEList: - # # call DataManager if targetSE in validReplicas: self.log.warn(f"Request to replicate {lfn} to an existing location: {targetSE}") continue res = self.dm.replicateAndRegister(lfn, targetSE, sourceSE=sourceSE, catalog=catalogs) if res["OK"]: - if lfn in res["Value"]["Successful"]: - if "replicate" in res["Value"]["Successful"][lfn]: - repTime = res["Value"]["Successful"][lfn]["replicate"] prString = f"file {lfn} replicated at {targetSE} in {repTime} s." if "register" in res["Value"]["Successful"][lfn]: - regTime = res["Value"]["Successful"][lfn]["register"] prString += " and registered in %s s." % regTime self.log.info(prString) else: - prString += " but failed to register" self.log.warn(prString) @@ -616,18 +610,15 @@ def dmTransfer(self, fromFTS=False): self.request.insertAfter(registerOperation, self.operation) else: - self.log.error("Failed to replicate", f"{lfn} to {targetSE}") opFile.Error = "Failed to replicate" else: - reason = res["Value"]["Failed"][lfn] self.log.error("Failed to replicate and register", f"File {lfn} at {targetSE}:", reason) opFile.Error = reason else: - opFile.Error = "DataManager error: %s" % res["Message"] self.log.error("DataManager error", res["Message"]) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py index 8452a3c2608..b2dcd209787 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py @@ -69,7 +69,6 @@ def setUp(self): self.assertEqual( self.mr.request.Status, 'Waiting' )""" def test__dmRemoval(self): - res = {"OK": True, "Value": {"Successful": {self.File.LFN: {"DIRACFileCatalog": True}}, "Failed": {}}} self.mr.dm.removeReplica.return_value = res @@ -110,7 +109,6 @@ def setUp(self): self.mr.ci = MagicMock() def test__dmTransfer(self): - successful = {} for sourceSE in self.op.sourceSEList: successful[sourceSE] = "dips://" + sourceSE.lower() + ":9148/DataManagement/StorageElement" + self.File.LFN @@ -141,7 +139,6 @@ def test__dmTransfer(self): self.assertEqual(self.mr.request.Status, "Waiting") def test__dmRemoval(self): - res = { "OK": True, "Value": {"Successful": {}, "Failed": {self.File.LFN: "Write access not permitted for this credential"}}, diff --git a/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py b/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py index 9e7e5b8d264..1cf4e6c66c6 100755 --- a/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py +++ b/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py @@ -16,7 +16,6 @@ class DataIntegrityClient(Client): """Client exposing the DataIntegrity Service.""" def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("DataManagement/DataIntegrity") self.dm = DataManager() diff --git a/src/DIRAC/DataManagementSystem/Client/DataManager.py b/src/DIRAC/DataManagementSystem/Client/DataManager.py index 7340b42f49b..2c9f32cfc8e 100644 --- a/src/DIRAC/DataManagementSystem/Client/DataManager.py +++ b/src/DIRAC/DataManagementSystem/Client/DataManager.py @@ -31,6 +31,7 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + # # RSCID def _isOlderThan(stringTime, days): """Check if a time stamp is older than a given number of days""" @@ -528,7 +529,6 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= if not res["OK"]: # We don't consider it a failure if the SE is not valid if not DErrno.cmpError(res, errno.EACCES): - accountingDict["TransferOK"] = 0 accountingDict["FinalStatus"] = "Failed" sendingResult = self.dataOpSender.sendData( @@ -818,7 +818,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= destPath = lfn for candidateSEName in possibleSourceSEs: - log.debug("Consider %s as a source" % candidateSEName) # Check that the candidate is active @@ -881,7 +880,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= # But that is the only way to make sure we are not replicating # over ourselves. for compatibleProtocol in replicationProtocols: - # Compare the urls to make sure we are not overwriting res = returnSingleResult(candidateSE.getURL(lfn, protocol=compatibleProtocol)) if not res["OK"]: @@ -893,7 +891,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= destURL = "" res = returnSingleResult(destStorageElement.getURL(destPath, protocol=compatibleProtocol)) if not res["OK"]: - # for some protocols, in particular srm # you might get an error because the file does not exist # which is exactly what we want @@ -942,7 +939,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= log.debug("Will try intermediate transfer from %s sources" % len(possibleIntermediateSEs)) for candidateSE in possibleIntermediateSEs: - res = returnSingleResult(candidateSE.getFile(lfn, localPath=localDir)) if not res["OK"]: log.debug("Error getting the file from %s" % candidateSE.name, res["Message"]) diff --git a/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py b/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py index 495410fb3f4..c852c9d21fc 100644 --- a/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py +++ b/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py @@ -12,7 +12,6 @@ class DirectoryListing: def __init__(self): - self.entries = [] def addFile(self, name, fileDict, repDict, numericid): diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3File.py b/src/DIRAC/DataManagementSystem/Client/FTS3File.py index 79a2e304758..0b687300af3 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3File.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3File.py @@ -59,7 +59,6 @@ class FTS3File(JSerializable): ] def __init__(self): - self.status = FTS3File.INIT_STATE self.attempt = 0 diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py index 29691afd9c7..ebc7cd19976 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py @@ -63,7 +63,6 @@ class FTS3Job(JSerializable): ] def __init__(self): - self.submitTime = None self.lastUpdate = None self.lastMonitor = None @@ -338,7 +337,6 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N fileIDsInTheJob = set() for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1): - # Again, this is relevant only for the very initial source # but code factorization is more important hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo) @@ -406,7 +404,6 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N allStageURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: - if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue @@ -434,7 +431,6 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N # * srcProto://myFile -> destProto://myFile if stageURL: - # We do not set a fileID in the metadata # such that we do not update the DB when monitoring stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID} @@ -608,7 +604,6 @@ def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): allTargetSURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: - if ftsFile.lfn in failedLFNs: log.debug("Not preparing transfer for file %s" % ftsFile.lfn) continue diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py b/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py index ac7c592e642..015064ec2bc 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py @@ -331,7 +331,6 @@ def _updateRmsOperationStatus(self): # { SE : [FTS3Files] } ftsFilesByTarget = {} for ftsFile in self.ftsFiles: - if ftsFile.status == "Defunct": log.info("File failed to transfer, setting it to failed in RMS", f"{ftsFile.lfn} {ftsFile.targetSE}") defunctRmsFileIDs.add(ftsFile.rmsFileID) @@ -401,7 +400,6 @@ class FTS3TransferOperation(FTS3Operation): """Class to be used for a Replication operation""" def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): - log = self._log.getSubLogger("_prepareNewJobs") filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) @@ -416,7 +414,6 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): filesGroupedByTarget = res["Value"] for targetSE, ftsFiles in filesGroupedByTarget.items(): - res = self._checkSEAccess(targetSE, "WriteAccess", vo=self.vo) if not res["OK"]: @@ -449,11 +446,9 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.items(): - # Checking whether we will need multiHop transfer multiHopSE = self.fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure(sourceSE, targetSE) if multiHopSE: - log.verbose("WLCG failure manifestation, use %s for multihop, max files per job is 1" % multiHopSE) # Check that we can write and read from it @@ -485,7 +480,6 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): maxFilesPerJob = 1 for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): - newJob = self._createNewJob( "Transfer", ftsFilesChunk, targetSE, sourceSE=sourceSE, multiHopSE=multiHopSE ) @@ -619,7 +613,6 @@ class FTS3StagingOperation(FTS3Operation): """Class to be used for a Staging operation""" def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): - log = gLogger.getSubLogger("_prepareNewJobs") filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) @@ -631,14 +624,12 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit) for targetSE, ftsFiles in filesGroupedByTarget.items(): - res = self._checkSEAccess(targetSE, "ReadAccess", vo=self.vo) if not res["OK"]: log.error(res) continue for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): - newJob = self._createNewJob("Staging", ftsFilesChunk, targetSE, sourceSE=targetSE) newJobs.append(newJob) diff --git a/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py b/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py index 39f542f0c9e..71d3cd09457 100644 --- a/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py +++ b/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py @@ -84,7 +84,6 @@ def transferAndRegisterFile( fileChecksum = fileMetaDict.get("Checksum", None) for se in destinationSEList: - # We put here some retry in case the problem comes from the FileCatalog # being unavailable. If it is, then the `hasAccess` call would fail, # and we would not make any failover request. So the only way is to wait a bit @@ -260,7 +259,6 @@ def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): catalog = [catalog] for cat in catalog: - register = Operation() register.Type = "RegisterFile" register.Catalog = cat diff --git a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py index 95422252b0e..7d5cddcd8c6 100644 --- a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py +++ b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py @@ -58,7 +58,6 @@ def __init__(self, client): self.ul_dc = DirectoryCompletion(self.ul_fs) def getPath(self, apath): - if apath.find("/") == 0: path = apath else: @@ -670,7 +669,6 @@ def do_ancestorset(self, args): print("Exception while adding ancestors: ", str(x)) def complete_ancestorset(self, text, line, begidx, endidx): - args = line.split() if len(args) == 1: @@ -1972,7 +1970,6 @@ def do_find(self, args): return if result["Value"]: - lfnList = result["Value"] if dirsOnly: listToPrint = {os.path.dirname(fullpath) for fullpath in lfnList} @@ -2270,7 +2267,6 @@ def do_repair(self, args): if __name__ == "__main__": - if len(sys.argv) > 2: print(FileCatalogClientCLI.__doc__) sys.exit(2) diff --git a/src/DIRAC/DataManagementSystem/Client/MetaQuery.py b/src/DIRAC/DataManagementSystem/Client/MetaQuery.py index 2302dab096e..b7105bc259d 100644 --- a/src/DIRAC/DataManagementSystem/Client/MetaQuery.py +++ b/src/DIRAC/DataManagementSystem/Client/MetaQuery.py @@ -52,7 +52,6 @@ class MetaQuery: def __init__(self, queryDict=None, typeDict=None): - self.__metaQueryDict = {} if queryDict is not None: self.__metaQueryDict = queryDict @@ -160,11 +159,9 @@ def setMetaQuery(self, queryList, metaTypeDict=None): return S_OK(metaDict) def getMetaQuery(self): - return self.__metaQueryDict def getMetaQueryAsJson(self): - return json.dumps(self.__metaQueryDict) def applyQuery(self, userMetaDict): @@ -192,7 +189,6 @@ def getTypedValue(value, mtype): return value for meta, value in self.__metaQueryDict.items(): - # Check if user dict contains all the requested meta data userValue = userMetaDict.get(meta, None) if userValue is None: diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py b/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py index 32e102eaf29..b1c3872e173 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py +++ b/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py @@ -13,7 +13,6 @@ class UtilitiesTestCase(unittest.TestCase): def setUp(self): - gLogger.setLevel("DEBUG") self.lfnDict = { diff --git a/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py b/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py index 44eedb91b98..7cf3ee9d741 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py +++ b/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py @@ -9,7 +9,6 @@ class DirCompletion(cmd.Cmd): - ulfs = UnixLikeFileSystem() dc = DirectoryCompletion(ulfs) @@ -48,6 +47,5 @@ def complete_ls(self, text, line, begidx, endidx): if __name__ == "__main__": - cli = DirCompletion() cli.cmdloop() diff --git a/src/DIRAC/DataManagementSystem/DB/FTS3DB.py b/src/DIRAC/DataManagementSystem/DB/FTS3DB.py index eb1b01f01cc..64c0f57d573 100644 --- a/src/DIRAC/DataManagementSystem/DB/FTS3DB.py +++ b/src/DIRAC/DataManagementSystem/DB/FTS3DB.py @@ -161,6 +161,7 @@ # We set it to `False` simply because we do not rely on the session cache. # Please see https://github.com/sqlalchemy/sqlalchemy/discussions/6159 for detailed discussion + ######################################################################## class FTS3DB: """ @@ -239,7 +240,6 @@ def persistOperation(self, operation): # because of the merge we have to explicitely set lastUpdate operation.lastUpdate = utc_timestamp() try: - # Merge it in case it already is in the DB operation = session.merge(operation) session.add(operation) @@ -269,7 +269,6 @@ def getOperation(self, operationID): session = self.dbSession(expire_on_commit=False) try: - operation = session.query(FTS3Operation).filter(getattr(FTS3Operation, "operationID") == operationID).one() session.commit() @@ -372,10 +371,8 @@ def updateFileStatus(self, fileStatusDict, ftsGUID=None): # (https://dev.mysql.com/doc/refman/5.7/en/innodb-deadlocks-handling.html) for fileID, valueDict in fileStatusDict.items(): - session = self.dbSession() try: - updateDict = {FTS3File.status: valueDict["status"]} # We only update error if it is specified @@ -433,9 +430,7 @@ def updateJobStatus(self, jobStatusDict): """ session = self.dbSession() try: - for jobID, valueDict in jobStatusDict.items(): - updateDict = {FTS3Job.status: valueDict["status"]} # We only update error if it is specified @@ -487,7 +482,6 @@ def cancelNonExistingJob(self, operationID, ftsGUID): """ session = self.dbSession() try: - # We update both the rows of the Jobs and the Files tables # having matching operationID and ftsGUID # https://docs.sqlalchemy.org/en/13/core/tutorial.html#multiple-table-updates @@ -541,7 +535,6 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned"): session = self.dbSession(expire_on_commit=False) try: - ftsOperations = [] # We need to do the select in two times because the join clause that makes the limit difficult @@ -603,7 +596,6 @@ def kickStuckOperations(self, limit=20, kickDelay=2): session = self.dbSession(expire_on_commit=False) try: - ftsOps = ( session.query(FTS3Operation.operationID) .filter( @@ -653,7 +645,6 @@ def kickStuckJobs(self, limit=20, kickDelay=2): session = self.dbSession(expire_on_commit=False) try: - ftsJobs = ( session.query(FTS3Job.jobID) .filter(FTS3Job.lastUpdate < (func.date_sub(utc_timestamp(), text("INTERVAL %d HOUR" % kickDelay)))) @@ -696,7 +687,6 @@ def deleteFinalOperations(self, limit=20, deleteDelay=180): session = self.dbSession(expire_on_commit=False) try: - ftsOps = ( session.query(FTS3Operation.operationID) .filter( @@ -741,7 +731,6 @@ def getOperationsFromRMSOpID(self, rmsOpID): session = self.dbSession(expire_on_commit=False) try: - operations = session.query(FTS3Operation).filter(FTS3Operation.rmsOpID == rmsOpID).all() session.commit() diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py index 77ede27369c..f60a41b01ab 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py @@ -8,7 +8,6 @@ class DatasetManager: - _tables = dict() _tables["FC_MetaDatasets"] = { "Fields": { @@ -100,7 +99,6 @@ def addDataset(self, datasets, credDict): return S_OK({"Successful": successful, "Failed": failed}) def __addDataset(self, datasetName, metaQuery, credDict, uid, gid): - result = self.__getMetaQueryParameters(metaQuery, credDict) if not result["OK"]: return result @@ -161,7 +159,6 @@ def _getDatasetDirectories(self, datasets): return dirDict def _findDatasets(self, datasets, connection=False): - connection = self._getConnection(connection) fullNames = [name for name in datasets if name.startswith("/")] shortNames = [name for name in datasets if not name.startswith("/")] @@ -183,7 +180,6 @@ def _findDatasets(self, datasets, connection=False): return S_OK(resultDict) def __findFullPathDatasets(self, datasets, connection): - dirDict = self._getDatasetDirectories(datasets) failed = {} successful = {} @@ -224,7 +220,6 @@ def __findFullPathDatasets(self, datasets, connection): return S_OK({"Successful": successful, "Failed": failed}) def __findNoPathDatasets(self, nodirDatasets, connection): - failed = {} successful = {} dsIDs = {} @@ -598,7 +593,6 @@ def getDatasetsInDirectory(self, dirID, verbose=False, connection=False): return S_OK(datasets) def __getDatasetDict(self, row): - resultDict = {} resultDict["DatasetID"] = int(row[0]) resultDict["MetaQuery"] = eval(row[1]) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py index cec86cf0af5..9beecbba051 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py @@ -19,7 +19,6 @@ def __init__(self, database=None): self.treeTable = "FC_DirectoryLevelTree" def getTreeType(self): - return "Directory" def findDir(self, path, connection=False): @@ -451,7 +450,6 @@ def recoverOrphanDirectories(self, credDict): parentDict = {} for dirID, parentID, _level in result["Value"]: - result = self.getDirectoryPath(dirID) if not result["OK"]: continue diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py index 489d49f24b8..fb0a1b5a942 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py @@ -16,7 +16,6 @@ def __init__(self, database=None): self.treeTable = "FC_DirectoryTree" def findDir(self, path): - req = "SELECT DirID from FC_DirectoryTree WHERE DirName='%s'" % path result = self.db._query(req) if not result["OK"]: @@ -42,7 +41,6 @@ def removeDir(self, path): return result def makeDir(self, path): - result = self.findDir(path) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py index 6b5f0bd5fef..263be23e02d 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py @@ -34,11 +34,9 @@ def findDirs(self, paths, connection=False): return S_ERROR("To be implemented on derived class") def makeDir(self, path): - return S_ERROR("To be implemented on derived class") def removeDir(self, path): - return S_ERROR("To be implemented on derived class") def getChildren(self, path, connection=False): @@ -893,7 +891,6 @@ def _getDirectoryLogicalSize(self, lfns, recursiveSum=True, connection=None): failed = {} treeTable = self.getTreeTable() for path in lfns: - if path == "/": req = "SELECT SUM(Size),COUNT(*) FROM FC_Files" reqDir = "SELECT count(*) FROM %s" % treeTable @@ -994,7 +991,6 @@ def _getDirectoryPhysicalSizeFromUsage_old(self, lfns, connection): successful = {} failed = {} for path in lfns: - if path == "/": req = "SELECT S.SEName, D.SESize, D.SEFiles FROM FC_DirectoryUsage as D, FC_StorageElements as S" req += " WHERE S.SEID=D.SEID" @@ -1141,7 +1137,6 @@ def __rebuildDirectoryUsageLeaves(self): empty = 0 for dirID in dirIDs: - count += 1 # Get the physical size diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py index 0f8abc8775d..f2436084b81 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py @@ -10,7 +10,6 @@ class DirectoryMetadata: def __init__(self, database=None): - self.db = database def setDatabase(self, database): diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py index 78851b7a6a3..fa882bf4298 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py @@ -12,7 +12,6 @@ class FileManager(FileManagerBase): - ###################################################### # # The all important _findFiles and _getDirectoryFiles methods @@ -239,7 +238,6 @@ def _getFileMetadataByID(self, fileIDs, connection=False): # def _insertFiles(self, lfns, uid, gid, connection=False): - connection = self._getConnection(connection) # Add the files failed = {} diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py index 58f918cd546..e96045eb5f5 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py @@ -738,7 +738,6 @@ def addReplica(self, lfns, connection=False): return S_OK({"Successful": successful, "Failed": failed}) def _addReplicas(self, lfns, connection=False): - connection = self._getConnection(connection) successful = {} res = self._findFiles(list(lfns), ["DirID", "FileID", "Size"], connection=connection) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py index 61d4d3e5549..8ae386b392d 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py @@ -6,7 +6,6 @@ class FileManagerFlat(FileManagerBase): - ###################################################### # # The all important _findFiles and _getDirectoryFiles methods diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index 90bddd6c872..563b8c574aa 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -81,7 +81,6 @@ def _findFileIDs(self, lfns, connection=False): successful[lfn] = fileId else: - # We separate the files by directory filesInDirDict = self._getFileDirectories(lfns) @@ -171,7 +170,6 @@ def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, files = {} for row in rows: - rowDict = dict(zip(fieldNames, row)) fileName = rowDict["FileName"] # Returns only the required metadata @@ -271,7 +269,6 @@ def _insertFiles(self, lfns, uid, gid, connection=False): # Prepare each file separately for lfn in lfns: - # Get all the info fileInfo = lfns[lfn] @@ -323,7 +320,6 @@ def _insertFiles(self, lfns, uid, gid, connection=False): if not result["OK"]: failed[lfn] = result["Message"] else: - fileID = result["Value"][0][0] successful[lfn] = lfns[lfn] @@ -506,7 +502,6 @@ def _insertReplicas(self, lfns, master=False, connection=False): # treat each file after each other for lfn in lfns.keys(): - fileID = lfns[lfn]["FileID"] seName = lfns[lfn]["SE"] @@ -523,7 +518,6 @@ def _insertReplicas(self, lfns, master=False, connection=False): # treat each replica of a file after the other # (THIS CANNOT WORK... WE ARE ONLY CAPABLE OF DOING ONE REPLICA PER FILE AT THE TIME) for seName in seList: - # get the SE id res = self.db.seManager.findSE(seName) if not res["OK"]: @@ -744,7 +738,6 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): # If there is an associated procedure, we go for it if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (fileID, paramValue)) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py index f1344fd6b25..1fcd4432553 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py @@ -14,7 +14,6 @@ class FileMetadata: def __init__(self, database=None): - self.db = database def setDatabase(self, database): @@ -461,7 +460,6 @@ def __createMetaSelection(self, value): queryList.append(("IN", query)) elif isinstance(value, dict): for operation, operand in value.items(): - # Prepare the escaped operand first if isinstance(operand, list): result = self.db._escapeValues(operand) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py index 50fafb01562..f394389fb99 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py @@ -9,7 +9,6 @@ class SEManagerDB(SEManagerBase): def _refreshSEs(self, connection=False): - req = "SELECT SEID,SEName FROM FC_StorageElements;" res = self.db._query(req) if not res["OK"]: diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py index f0b8d11f946..2becaf53e27 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py @@ -76,7 +76,6 @@ def getPathPermissions(self, paths, credDict): # For all the paths that exist, check the write permission if paths: - res = super().getPathPermissions(paths, credDict) if not res["OK"]: return res diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py index d09c097d031..3bb33002504 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py @@ -182,7 +182,6 @@ def getPathPermissions(self, lfns, credDict): successful = {} failed = {} for filename in lfns: - if filename not in fileTree: failed[filename] = "File not found" continue @@ -265,7 +264,6 @@ class BaseCaseMixin: side_effect=mock_getAllGroups, ) def setUp(self, _a, _b): - global directoryTree global fileTree # A dictionary of directories. The keys are path, @@ -311,7 +309,6 @@ def compareResult(self): ("Existing", self.existingRet, self.expectedExistingRet), ("NonExisting", self.nonExistingRet, self.expectedNonExistingRet), ]: - self.assertTrue(real, "The method was not run") self.assertTrue(expected, "No expected results given") @@ -1595,7 +1592,6 @@ def test_addReplica(self): if __name__ == "__main__": - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestNonExistingUser) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestAdminGrpAnonUser)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestAdminGrpAdminUser)) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py index 0db2bfeaf52..55de76566c5 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py @@ -62,7 +62,6 @@ def setConfig(self, databaseConfig): ("dmeta", "DirectoryMetadata"), ("fmeta", "FileMetadata"), ]: - result = self.__loadCatalogComponent(componentType, databaseConfig[componentType]) if not result["OK"]: return result @@ -1176,7 +1175,6 @@ def _checkAdminPermission(self, credDict): return self.securityManager.hasAdminAccess(credDict) def _checkPathPermissions(self, operation, lfns, credDict): - res = checkArgumentFormat(lfns) if not res["OK"]: return res diff --git a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py index 27becd973b2..950cca94a8d 100644 --- a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py @@ -72,7 +72,6 @@ def initializeHandler(cls, serviceInfoDict): and "Aws_access_key_id" in storageParam and "Aws_secret_access_key" in storageParam ): - cls._S3Storages[seName] = storagePlugin log.debug("Add %s to the list of usable S3 storages" % seName) break diff --git a/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py b/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py index 18ba26add6e..329336ae055 100644 --- a/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py +++ b/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py @@ -59,7 +59,6 @@ def selectUniqueSource(ftsFiles, fts3Plugin, allowedSources=None): failedFiles = {} for ftsFile in ftsFiles: - # If we failed to get the replicas, add the FTS3File to the dictionary if ftsFile.lfn in filteredReplicas["Failed"]: errMsg = filteredReplicas["Failed"][ftsFile.lfn] @@ -205,7 +204,6 @@ def chooseFTS3Server(self): attempt = 0 while not fts3Server and attempt < self._maxAttempts: - fts3Server = self._policyMethod(attempt) res = self._getFTSServerStatus(fts3Server) diff --git a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py index f870e0224b9..08a759950ac 100644 --- a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py +++ b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py @@ -17,7 +17,6 @@ class HttpStorageAccessHandler(server.BaseHTTPRequestHandler): - register = DictCache() basePath = "" diff --git a/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py b/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py index a654a8136cb..5e6437f6858 100644 --- a/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py +++ b/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py @@ -50,7 +50,6 @@ def setUp(self): self.allFiles = [self.f1, self.f2, self.f3, self.f4] def test_01_groupFilesByTarget(self): - # empty input self.assertTrue(groupFilesByTarget([])["Value"] == {}) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py index 9787254d19d..d154238fdea 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py @@ -108,12 +108,10 @@ def main(): reason = "Forced with dirac-admin-ban-se by %s" % userName for se, seOptions in res["Value"].items(): - resW = resC = resR = {"OK": False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: - if seOptions["ReadAccess"] == "Banned": gLogger.notice("Read access already banned", se) resR["OK"] = True @@ -124,7 +122,6 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR["OK"]: @@ -135,7 +132,6 @@ def main(): # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: - if seOptions["WriteAccess"] == "Banned": gLogger.notice("Write access already banned", se) resW["OK"] = True @@ -146,7 +142,6 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW["OK"]: @@ -157,7 +152,6 @@ def main(): # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: - if seOptions["CheckAccess"] == "Banned": gLogger.notice("Check access already banned", se) resC["OK"] = True @@ -168,7 +162,6 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: @@ -179,7 +172,6 @@ def main(): # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: - if seOptions["RemoveAccess"] == "Banned": gLogger.notice("Remove access already banned", se) resC["OK"] = True @@ -190,7 +182,6 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py index 76cd5303240..9d4938e0173 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py @@ -57,7 +57,6 @@ def main(): requestOperation = "RemoveFile" for lfnList in breakListIntoChunks(lfns, 100): - oRequest = Request() requestName = "{}_{}".format( md5(repr(time.time()).encode()).hexdigest()[:16], diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py index 48baf1713b2..84eb1293aa7 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py @@ -131,7 +131,6 @@ def main(): fromSE = list(seForSeBases) targetSE = list(seForSeBases) else: # he specified at least source of dest - # if bidirection, source and target should be the same if bidirection: if not fromSE and targetSE: # we gave target, but no source @@ -176,7 +175,6 @@ def main(): # For each source and destination, generate the url pair, and the compatible third party protocols for src, dst in ((x, y) for x in fromSE for y in targetSE): - if ftsTab: try: # breakpoint() diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py index ca284d1200c..ca6d9bcf82b 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py @@ -93,7 +93,6 @@ def main(): vo = result["Value"] for se, statusDict in res["Value"].items(): - # Check if the SE is allowed for the user VO if not allVOsFlag: voList = gConfig.getValue("/Resources/StorageElements/%s/VO" % se, []) diff --git a/src/DIRAC/FrameworkSystem/API/AuthHandler.py b/src/DIRAC/FrameworkSystem/API/AuthHandler.py index 1bef460b7e9..4132dad0b69 100644 --- a/src/DIRAC/FrameworkSystem/API/AuthHandler.py +++ b/src/DIRAC/FrameworkSystem/API/AuthHandler.py @@ -432,13 +432,13 @@ def __researchDIRACGroup(self, extSession, chooseScope, state): # Read requested groups by DIRAC client or user firstRequest.addScopes(chooseScope) # Read already authed user - username = extSession["authed"]["username"] + userName = extSession["authed"]["username"] # Requested arguments in first request provider = firstRequest.provider - self.log.debug("Next groups has been found for %s:" % username, ", ".join(firstRequest.groups)) + self.log.debug("The following groups found", f"for {userName}: {', '.join(firstRequest.groups)}") - # Researche Group - result = getGroupsForUser(username) + # Research Group + result = getGroupsForUser(userName) if not result["OK"]: return None, self.server.handle_response( payload=getHTML("server error", theme="error", info=result["Message"]), delSession=True @@ -453,12 +453,12 @@ def __researchDIRACGroup(self, extSession, chooseScope, state): payload=getHTML( "groups not found.", theme="error", - info=f"No groups found for {username} and for {provider} Identity Provider.", + info=f"No groups found for {userName} and for {provider} Identity Provider.", ), delSession=True, ) - self.log.debug("The state of %s user groups has been checked:" % username, pprint.pformat(validGroups)) + self.log.debug("The state of %s user groups has been checked:" % userName, pprint.pformat(validGroups)) # If group already defined in first request, just return it if firstRequest.groups: diff --git a/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py b/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py index 488d882d658..bd7ac3e2062 100644 --- a/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py @@ -18,7 +18,6 @@ class ProxyRenewalAgent(AgentModule): def initialize(self): - requiredLifeTime = self.am_getOption("MinimumLifeTime", 3600) renewedLifeTime = self.am_getOption("RenewedLifeTime", 54000) mailFrom = self.am_getOption("MailFrom", DEFAULT_MAIL_FROM) diff --git a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py index 8ff334afd07..597f153f989 100644 --- a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py @@ -737,7 +737,6 @@ def test_checkURLs_2(self): "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", side_effect=mockComponentSection, ): - res = self.restartAgent.checkURLs() self.assertTrue(res["OK"]) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 9474f526bce..704d60ad0b7 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -2338,7 +2338,6 @@ def installTornado(self): # Now do the actual installation try: - self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, "run") diff --git a/src/DIRAC/FrameworkSystem/Client/NotificationClient.py b/src/DIRAC/FrameworkSystem/Client/NotificationClient.py index a396ddf7264..57c43044bab 100644 --- a/src/DIRAC/FrameworkSystem/Client/NotificationClient.py +++ b/src/DIRAC/FrameworkSystem/Client/NotificationClient.py @@ -27,7 +27,6 @@ def sendMail(self, addresses, subject, body, fromAddress=None, localAttempt=True addresses = [addresses] if isinstance(addresses, str) else list(addresses) for address in addresses: - if localAttempt: try: m = Mail() diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py index f8fa303167c..76b995d45b0 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py @@ -10,7 +10,6 @@ class CLIParams: - proxyLifeTime = 86400 diracGroup = False proxyStrength = DEFAULT_PROXY_STRENGTH diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py b/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py index f0528033a08..cd43dca336e 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py @@ -7,7 +7,6 @@ class CLIParams: - proxyLifeTime = 2592000 certLoc = False keyLoc = False diff --git a/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py b/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py index 388b78a68fb..d4231831cda 100644 --- a/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py +++ b/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py @@ -9,7 +9,6 @@ class SecurityLogClient: - __securityLogStore = [] def __init__(self): diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 1c8fb304352..ba3832e994c 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -30,7 +30,6 @@ class SystemAdministratorClientCLI(CLI): """Line oriented command interpreter for administering DIRAC components""" def __init__(self, host=None): - CLI.__init__(self) # Check if Port is given self.host = None @@ -1319,7 +1318,6 @@ def do_showall(self, args): print("\n %d out of %d hosts did not respond" % (len(silentHosts), totalHosts)) def default(self, args): - command = args.split()[0] if command in ["ls", "cat", "pwd", "chown", "chmod", "chgrp", "id", "date", "uname", "cp", "mv", "scp"]: self.do_exec(args) diff --git a/src/DIRAC/FrameworkSystem/Client/TokenManagerClient.py b/src/DIRAC/FrameworkSystem/Client/TokenManagerClient.py index 963ff25e7da..349ddd40625 100644 --- a/src/DIRAC/FrameworkSystem/Client/TokenManagerClient.py +++ b/src/DIRAC/FrameworkSystem/Client/TokenManagerClient.py @@ -7,6 +7,7 @@ from DIRAC.Core.Base.Client import Client, createClient from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory +from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import OAuth2Token gTokensSync = ThreadSafe.Synchronizer() @@ -26,16 +27,16 @@ def __init__(self, **kwargs): @gTokensSync def getToken( self, - username: str, + userName: str, userGroup: str = None, scope: str = None, audience: str = None, identityProvider: str = None, requiredTimeLeft: int = 0, ): - """Get an access token for a user/group. + """Get an access token for a user/group keeping the local cache - :param username: user name + :param userName: user name :param userGroup: group name :param scope: scope :param audience: audience @@ -55,36 +56,35 @@ def getToken( return result idpObj = result["Value"] - if userGroup and (result := idpObj.getGroupScopes(userGroup))["OK"]: + if userGroup and (result := idpObj.getGroupScopes(userGroup)): # What scope correspond to the requested group? - scope = list(set((scope or []) + result["Value"])) + scope = list(set((scope or []) + result)) # Set the scope idpObj.scope = " ".join(scope) # Let's check if there are corresponding tokens in the cache - cacheKey = (username, idpObj.scope, audience, identityProvider) + cacheKey = (userName, idpObj.scope, audience, identityProvider) if self.__tokensCache.exists(cacheKey, requiredTimeLeft): # Well we have a fresh record containing a Token object token = self.__tokensCache.get(cacheKey) # Let's check if the access token is fresh if not token.is_expired(requiredTimeLeft): return S_OK(token) - # It seems that it is no longer valid for us, but whether there is a refresh token? - if token.get("refresh_token"): - # Okay, so we can try to refresh tokens - if (result := idpObj.refreshToken(token["refresh_token"]))["OK"]: - # caching new tokens - self.__tokensCache.add( - cacheKey, - token.get_claim("exp", "refresh_token") or self.DEFAULT_RT_EXPIRATION_TIME, - result["Value"], - ) - return result - self.log.verbose(f"Failed to get token on client's side: {result['Message']}") - # Let's try to revoke broken token - idpObj.revokeToken(token["refresh_token"]) - - return self.executeRPC( - username, userGroup, scope, audience, identityProvider, requiredTimeLeft, call="getToken" + + result = self.executeRPC( + userName, userGroup, scope, audience, identityProvider, requiredTimeLeft, call="getToken" ) + + if result["OK"]: + token = OAuth2Token(dict(result["Value"])) + self.__tokensCache.add( + cacheKey, + token.get_claim("exp", "refresh_token") or self.DEFAULT_RT_EXPIRATION_TIME, + token, + ) + + return result + + +gTokenManager = TokenManagerClient() diff --git a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py index c0dd4ab3fec..900a438a886 100644 --- a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +++ b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py @@ -316,7 +316,6 @@ def __init__(self): raise Exception("Can't create tables: %s" % result["Message"]) def __initializeConnection(self, dbPath): - result = getDBParameters(dbPath) if not result["OK"]: raise Exception("Cannot get database parameters: %s" % result["Message"]) @@ -463,17 +462,17 @@ def __filterInstalledComponentsFields(self, session, matchFields=None): matchFields = {} componentKeys = {} - for (key, val) in matchFields.items(): + for key, val in matchFields.items(): if "Component." in key: componentKeys[key.replace("Component.", "")] = val hostKeys = {} - for (key, val) in matchFields.items(): + for key, val in matchFields.items(): if "Host." in key: hostKeys[key.replace("Host.", "")] = val selfKeys = {} - for (key, val) in matchFields.items(): + for key, val in matchFields.items(): if "Component." not in key and "Host." not in key: selfKeys[key] = val diff --git a/src/DIRAC/FrameworkSystem/DB/NotificationDB.py b/src/DIRAC/FrameworkSystem/DB/NotificationDB.py index ee6b2642b3c..95c7c3fe302 100644 --- a/src/DIRAC/FrameworkSystem/DB/NotificationDB.py +++ b/src/DIRAC/FrameworkSystem/DB/NotificationDB.py @@ -471,7 +471,6 @@ def __sendMailToUser(self, user, subject, message): return result def getAlarms(self, condDict={}, sortList=False, start=0, limit=0, modifiedAfter=None): - condSQL = [] for field in self.__alarmQueryFields: if field in condDict: @@ -802,7 +801,6 @@ def markNotificationsSeen(self, user, seen=True, msgIds=False): return self._update(updateSQL) def getNotifications(self, condDict={}, sortList=False, start=0, limit=0): - condSQL = [] for field in self.__notificationQueryFields: if field in condDict: diff --git a/src/DIRAC/FrameworkSystem/DB/ProxyDB.py b/src/DIRAC/FrameworkSystem/DB/ProxyDB.py index baf79f26bb9..d2dc5558b1a 100755 --- a/src/DIRAC/FrameworkSystem/DB/ProxyDB.py +++ b/src/DIRAC/FrameworkSystem/DB/ProxyDB.py @@ -34,7 +34,6 @@ class ProxyDB(DB): - NOTIFICATION_TIMES = [2592000, 1296000] def __init__(self, useMyProxy=False, mailFrom=None, parentLogger=None): @@ -536,7 +535,6 @@ def getProxyStrength(self, userDN, userGroup=None, vomsAttr=None): providers = retVal["Value"] providers.append("Certificate") for proxyProvider in providers: - retVal = self.__getPemAndTimeLeft(userDN, userGroup, vomsAttr=vomsAttr, proxyProvider=proxyProvider) if retVal["OK"]: pemData = retVal["Value"][0] diff --git a/src/DIRAC/FrameworkSystem/DB/TokenDB.py b/src/DIRAC/FrameworkSystem/DB/TokenDB.py index 1bd3eaa5075..3244e68e0f5 100644 --- a/src/DIRAC/FrameworkSystem/DB/TokenDB.py +++ b/src/DIRAC/FrameworkSystem/DB/TokenDB.py @@ -21,7 +21,7 @@ class Token(Model, OAuth2TokenMixin): - """This class describe token fields""" + """This class describes token fields""" __tablename__ = "Token" __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} @@ -30,8 +30,8 @@ class Token(Model, OAuth2TokenMixin): # https://stackoverflow.com/questions/1827063/mysql-error-key-specification-without-a-key-length id = Column(Integer, autoincrement=True, primary_key=True) # Unique token ID kid = Column(String(255)) # Unique secret key ID for token encryption - user_id = Column(String(255)) # User identificator that registred in an identity provider, token owner - provider = Column(String(255)) # Provider name registred in DIRAC + user_id = Column(String(255)) # User identifier registered in an identity provider, token owner + provider = Column(String(255)) # Provider name registered in DIRAC expires_at = Column(Integer, nullable=False, default=0) # When the access token is expired access_token = Column(Text, nullable=False) refresh_token = Column(Text, nullable=False) @@ -101,7 +101,7 @@ def updateToken(self, token: dict, userID: str, provider: str, rt_expired_in: in :return: S_OK(list)/S_ERROR() -- return old tokens that should be revoked. """ if not token["refresh_token"]: - return S_ERROR("Cannot store absent refresh token.") + return S_ERROR("Cannot store token without a refresh token.") # Let's collect the necessary attributes of the token token["user_id"] = userID diff --git a/src/DIRAC/FrameworkSystem/DB/UserProfileDB.py b/src/DIRAC/FrameworkSystem/DB/UserProfileDB.py index b0908f9b99e..b642c26ab63 100755 --- a/src/DIRAC/FrameworkSystem/DB/UserProfileDB.py +++ b/src/DIRAC/FrameworkSystem/DB/UserProfileDB.py @@ -395,7 +395,6 @@ def storeVarByUserId(self, userIds, profileName, varName, data, perms): return self._update(updateSQL) def setUserVarPermsById(self, userIds, profileName, varName, perms): - result = self._escapeString(profileName) if not result["OK"]: return result diff --git a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py index 3d47e28bc15..5567c06a4ce 100644 --- a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py @@ -17,7 +17,6 @@ class ProxyManagerHandlerMixin: - __maxExtraLifeFactor = 1.5 __proxyDB = None diff --git a/src/DIRAC/FrameworkSystem/Service/SecurityLoggingHandler.py b/src/DIRAC/FrameworkSystem/Service/SecurityLoggingHandler.py index a6906e23bad..1d2c5255315 100644 --- a/src/DIRAC/FrameworkSystem/Service/SecurityLoggingHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/SecurityLoggingHandler.py @@ -40,7 +40,6 @@ def initializeSecurityLoggingHandler(serviceInfo): class SecurityLoggingHandler(RequestHandler): - types_logAction = [(list, tuple)] def export_logAction(self, secMsg): diff --git a/src/DIRAC/FrameworkSystem/Service/TokenManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/TokenManagerHandler.py index 4a8e6b787f1..4b4839afb5c 100644 --- a/src/DIRAC/FrameworkSystem/Service/TokenManagerHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/TokenManagerHandler.py @@ -45,7 +45,6 @@ class TokenManagerHandler(TornadoService): - DEFAULT_AUTHORIZATION = ["authenticated"] DEFAULT_RT_EXPIRATION_TIME = 24 * 3600 @@ -177,7 +176,7 @@ def __checkProperties(self, requestedUserDN: str, requestedUserGroup: str): @gTokensSync def export_getToken( self, - username: str, + userName: str, userGroup: str = None, scope: str = None, audience: str = None, @@ -191,7 +190,7 @@ def export_getToken( * LimitedDelegation <- permits downloading only limited tokens * PrivateLimitedDelegation <- permits downloading only limited tokens for one self - :paarm username: user name + :param userName: user name :param userGroup: user group :param scope: requested scope :param audience: requested audience @@ -211,39 +210,25 @@ def export_getToken( return result idpObj = result["Value"] - if userGroup and (result := idpObj.getGroupScopes(userGroup))["OK"]: + if userGroup and (result := idpObj.getGroupScopes(userGroup)): # What scope correspond to the requested group? - scope = list(set((scope or []) + result["Value"])) + scope = list(set((scope or []) + result)) # Set the scope idpObj.scope = " ".join(scope) - # Let's check if there are corresponding tokens in the cache - cacheKey = (username, idpObj.scope, audience, identityProvider) + # Let's check if there is a corresponding token in the cache + cacheKey = (userName, idpObj.scope, audience, identityProvider) if self.__tokensCache.exists(cacheKey, requiredTimeLeft): # Well we have a fresh record containing a Token object token = self.__tokensCache.get(cacheKey) # Let's check if the access token is fresh if not token.is_expired(requiredTimeLeft): return S_OK(token) - # It seems that it is no longer valid for us, but whether there is a refresh token? - if token.get("refresh_token"): - # Okay, so we can try to refresh tokens - if (result := idpObj.refreshToken(token["refresh_token"]))["OK"]: - # caching new tokens - self.__tokensCache.add( - cacheKey, - result["Value"].get_claim("exp", "refresh_token") or self.DEFAULT_RT_EXPIRATION_TIME, - result["Value"], - ) - return result - self.log.verbose(f"Failed to get token with cached tokens: {result['Message']}") - # Let's try to revoke broken token - idpObj.revokeToken(token["refresh_token"]) err = [] - # The cache did not help, so let's make an exchange token - result = Registry.getDNForUsername(username) + # No luck so far, let's refresh the token stored in the database + result = Registry.getDNForUsername(userName) if not result["OK"]: return result for dn in result["Value"]: @@ -257,8 +242,8 @@ def export_getToken( idpObj.token = result["Value"] result = self.__checkProperties(dn, userGroup) if result["OK"]: - # exchange token with requested scope - result = idpObj.exchangeToken() + # refresh token with requested scope + result = idpObj.refreshToken() if result["OK"]: # caching new tokens self.__tokensCache.add( @@ -267,10 +252,10 @@ def export_getToken( result["Value"], ) return result - # Not find any token associated with the found user ID - err.append(result.get("Message", "No token found for %s." % uid)) + # Not found any token associated with the found user ID + err.append(result.get("Message", "No token found for %s" % uid)) # Collect all errors when trying to get a token, or if no user ID is registered - return S_ERROR("; ".join(err or ["No user ID found for %s" % username])) + return S_ERROR("; ".join(err or ["No user ID found for %s" % userName])) def export_deleteToken(self, userDN: str): """Delete a token from the DB diff --git a/src/DIRAC/FrameworkSystem/Service/TornadoBundleDeliveryHandler.py b/src/DIRAC/FrameworkSystem/Service/TornadoBundleDeliveryHandler.py index 791ee581458..04a0a0e602d 100644 --- a/src/DIRAC/FrameworkSystem/Service/TornadoBundleDeliveryHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/TornadoBundleDeliveryHandler.py @@ -9,7 +9,6 @@ class TornadoBundleDeliveryHandler(BundleDeliveryHandlerMixin, TornadoService): - types_streamToClient = [] def export_streamToClient(self, fileId): diff --git a/src/DIRAC/FrameworkSystem/Utilities/RabbitMQSynchronizer.py b/src/DIRAC/FrameworkSystem/Utilities/RabbitMQSynchronizer.py index 54b0a9db425..9b5a2dc0f42 100644 --- a/src/DIRAC/FrameworkSystem/Utilities/RabbitMQSynchronizer.py +++ b/src/DIRAC/FrameworkSystem/Utilities/RabbitMQSynchronizer.py @@ -17,7 +17,6 @@ class RabbitMQSynchronizer: def __init__(self): - # Warm up local CS # I am not sure whether it is needed but # it was used in DIRAC.ResourceStatusSystem.Utilities.Synchronizer diff --git a/src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py b/src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py index 68adde0bc26..0c650d96526 100644 --- a/src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py +++ b/src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py @@ -121,7 +121,7 @@ def generateProxyOrToken( :param client: instance of the IdP client :param grant_type: authorization grant type (unused) - :param str user: user identificator + :param str user: user identifier :param str scope: requested scope :param expires_in: when the token should expire (unused) :param bool include_refresh_token: to include refresh token (unused) @@ -282,7 +282,7 @@ def parseIdPAuthorizationResponse(self, response, session): :return: S_OK(dict)/S_ERROR() """ providerName = session.pop("Provider") - sLog.debug("Try to parse authentification response from %s:\n" % providerName, pprint.pformat(response)) + sLog.debug(f"Try to parse authentication response from {providerName}:\n{pprint.pformat(response)}") # Parse response result = self.idps.getIdProvider(providerName) if not result["OK"]: @@ -297,10 +297,10 @@ def parseIdPAuthorizationResponse(self, response, session): credDict, payload = result["Value"] sLog.debug("Read profile:", pprint.pformat(credDict)) - # Is ID registred? + # Is ID registered? result = getUsernameForDN(credDict["DN"]) if not result["OK"]: - comment = f"ID {credDict['ID']} is not registred in DIRAC. " + comment = f"ID {credDict['ID']} is not registered in DIRAC. " payload.update(idpObj.getUserProfile().get("Value", {})) result = self.__registerNewUser(providerName, payload) @@ -428,7 +428,7 @@ def validate_consent_request(self, request, provider=None): ) def validateIdentityProvider(self, request, provider): - """Check if identity provider registred in DIRAC + """Check if identity provider registered in DIRAC :param object request: request :param str provider: provider name @@ -441,12 +441,12 @@ def validateIdentityProvider(self, request, provider): # Find identity provider for group groupProvider = getIdPForGroup(request.group) if request.groups else None - # If requested access token for group that is not registred in any identity provider + # If requested access token for group that is not registered in any identity provider # or the requested provider does not match the group return error if request.group and not groupProvider and "proxy" not in request.scope: raise Exception("The %s group belongs to the VO that is not tied to any Identity Provider." % request.group) - sLog.debug("Check if %s identity provider registred in DIRAC.." % request.provider) + sLog.debug(f"Check if {request.provider} identity provider registered in DIRAC...") # Research supported IdPs result = getProvidersForInstance("Id") if not result["OK"]: diff --git a/src/DIRAC/FrameworkSystem/private/authorization/grants/AuthorizationCode.py b/src/DIRAC/FrameworkSystem/private/authorization/grants/AuthorizationCode.py index 3391aab1746..8ba48c6a6ef 100644 --- a/src/DIRAC/FrameworkSystem/private/authorization/grants/AuthorizationCode.py +++ b/src/DIRAC/FrameworkSystem/private/authorization/grants/AuthorizationCode.py @@ -8,7 +8,7 @@ class OAuth2Code(dict): - """This class describe Authorization Code object""" + """This class describes Authorization Code object""" def __init__(self, params): """C'or""" diff --git a/src/DIRAC/FrameworkSystem/private/authorization/utils/Utilities.py b/src/DIRAC/FrameworkSystem/private/authorization/utils/Utilities.py index b9280f94729..e36002200d3 100644 --- a/src/DIRAC/FrameworkSystem/private/authorization/utils/Utilities.py +++ b/src/DIRAC/FrameworkSystem/private/authorization/utils/Utilities.py @@ -134,7 +134,6 @@ def getHTML(title, info=None, body=None, style=None, state=None, theme=None, ico dom.div(dom.img(src=diracLogo, cls="card-img px-2"), cls="col-md-6 my-3") # Information card with dom.div(cls="col-md-6 my-3"): - # Show response state number if state and state != 200: dom.div(dom.h1(state, cls="text-center badge bg-%s text-wrap" % theme), cls="row py-2") diff --git a/src/DIRAC/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py b/src/DIRAC/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py index 660e97b9a89..14729a9b213 100644 --- a/src/DIRAC/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py +++ b/src/DIRAC/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py @@ -36,7 +36,6 @@ def test_createLogRecord(sMsg, sVarMsg, exc_info, expected): levels = gLogger.getAllPossibleLevels() for level in levels: for logger, logInfo in logDict.items(): - # createLogRecord is the method in charge of creating the log record # debug, ..., always methods wrap the following method # we use logLevels to get the int value corresponding to the level name diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py index 572f70933b5..dee95df3f19 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py @@ -20,7 +20,6 @@ class Params: - limited = False proxyPath = False proxyLifeTime = 86400 diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py index 0bed27cf749..72bf8ff01d8 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_users_with_proxy.py @@ -29,7 +29,6 @@ class Params: - limited = False proxyPath = False proxyLifeTime = 3600 diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py index d9910c84aec..82c63ec4d30 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_info.py @@ -25,7 +25,6 @@ class Params: - proxyLoc = False vomsEnabled = True csEnabled = True diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py index 36732bc4b14..c13c75da126 100755 --- a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py +++ b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py @@ -24,7 +24,6 @@ class Params(ProxyGeneration.CLIParams): - addVOMSExt = False uploadProxy = True uploadPilot = False diff --git a/src/DIRAC/Interfaces/API/Job.py b/src/DIRAC/Interfaces/API/Job.py index 2c8c9fbe09c..f8dc66225b3 100755 --- a/src/DIRAC/Interfaces/API/Job.py +++ b/src/DIRAC/Interfaces/API/Job.py @@ -985,7 +985,6 @@ def _toXML(self): return self.workflow.toXML() def _handleParameterSequences(self, paramsDict, arguments): - for pName in self.parameterSeqs: if pName in paramsDict: if pName == "InputSandbox": diff --git a/src/DIRAC/Interfaces/API/test/Test_JobAPI.py b/src/DIRAC/Interfaces/API/test/Test_JobAPI.py index 0f757090518..ff6c3e612cb 100644 --- a/src/DIRAC/Interfaces/API/test/Test_JobAPI.py +++ b/src/DIRAC/Interfaces/API/test/Test_JobAPI.py @@ -35,7 +35,6 @@ def test_basicJob(): def test_SimpleParametricJob(): - job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") @@ -84,7 +83,6 @@ def test_SimpleParametricJob(): ], ) def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): - job = Job() job.setExecutable("myExec") job.setLogLevel("DEBUG") diff --git a/src/DIRAC/Interfaces/Utilities/DCommands.py b/src/DIRAC/Interfaces/Utilities/DCommands.py index 9f8d7b41f83..f81bc4df6ef 100644 --- a/src/DIRAC/Interfaces/Utilities/DCommands.py +++ b/src/DIRAC/Interfaces/Utilities/DCommands.py @@ -45,7 +45,6 @@ def _getProxyInfo(proxyPath=False): def listFormatPretty(summaries, headers=None, sortKeys=None): - records = [] for _k, i in sortKeys: records.append([str(x) for x in summaries[i]]) @@ -268,7 +267,6 @@ def fillMinimal(self): def createMinimalConfig(configDir=os.path.expanduser("~/.dirac"), configFilename="dcommands.conf"): - dconfig = DConfig(configDir, configFilename) modified = dconfig.fillMinimal() diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py index c3e8025c9c3..c24d6e6e350 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilot_output.py @@ -27,7 +27,6 @@ def main(): errorList = [] for job in args: - try: job = int(job) except Exception: diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py index 04ff4081866..815347932a1 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_job_pilots.py @@ -46,7 +46,6 @@ def main(): errorList = [] for job in args: - try: job = int(job) except ValueError: diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py index c57196e7350..2e2f61cb507 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_logging_info.py @@ -44,7 +44,6 @@ def main(): errorList = [] for gridID in args: - result = diracAdmin.getPilotLoggingInfo(gridID) if not result["OK"]: errorList.append((gridID, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py index f7b20c269e0..fed8f0bf111 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_get_pilot_output.py @@ -28,7 +28,6 @@ def main(): errorList = [] for gridID in args: - result = diracAdmin.getPilotOutput(gridID) if not result["OK"]: errorList.append((gridID, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py index 97b972ea7da..05c3a27e611 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_reset_job.py @@ -27,7 +27,6 @@ def main(): errorList = [] for job in args: - try: job = int(job) except Exception as x: diff --git a/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py index 56df433be93..19c6b5b0790 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_admin_site_info.py @@ -32,7 +32,6 @@ def main(): errorList = [] for site in args: - result = diracAdmin.getSiteSection(site, printOutput=True) if not result["OK"]: errorList.append((site, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py index e8f9792851a..7ea6d922bd0 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_normalized_queue_length.py @@ -27,7 +27,6 @@ def main(): exitCode = 0 for ceUniqueID in args: - normCPU = queueNormalizedCPU(ceUniqueID) if not normCPU["OK"]: diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py index 86cce570044..b08eef53f68 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_get_queue_normalization.py @@ -25,7 +25,6 @@ def main(): from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getQueueNormalization for ceUniqueID in args: - cpuNorm = getQueueNormalization(ceUniqueID) if not cpuNorm["OK"]: diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py index b68fbe1cafb..ff04d1a63e4 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -60,7 +60,6 @@ def main(): errorList = [] for job in parseArguments(args): - result = dirac.getJobAttributes(job, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py index 432cdc46e0e..029f40b0d5c 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_input.py @@ -35,7 +35,6 @@ def main(): outputDir = v for job in parseArguments(args): - result = dirac.getInputSandbox(job, outputDir=outputDir) if result["OK"]: if os.path.exists("InputSandbox%s" % job): diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py index d7fa8181ba3..872d4e135f7 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py @@ -55,7 +55,6 @@ def main(): errorList = [] for job in parseArguments(args): - result = dirac.getJobJDL(job, original=original, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py index 07990e28b75..2007fd54c56 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output.py @@ -92,7 +92,6 @@ def main(): todoJobs = [job for job in jobs if job not in doneJobs] for job in todoJobs: - result = dirac.getOutputSandbox(job, outputDir=outputDir) jobDir = str(job) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py index 25a198fffb5..e1184e9e7dc 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_output_data.py @@ -29,7 +29,6 @@ def main(): outputDir = v for job in parseArguments(args): - result = dirac.getJobOutputData(job, destinationDir=outputDir) if result["OK"]: print("Job %s output data retrieved" % (job)) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py index 4c58ebab8b7..bc7affcf1ec 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_logging_info.py @@ -41,7 +41,6 @@ def main(): errorList = [] for job in parseArguments(args): - result = dirac.getJobLoggingInfo(job, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py index a66367ebc8f..7984c1e9988 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py @@ -47,7 +47,6 @@ def main(): errorList = [] for job in parseArguments(args): - result = dirac.getJobParameters(job, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py index ef51d5fbf1a..86e04d72fed 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_peek.py @@ -26,7 +26,6 @@ def main(): errorList = [] for job in parseArguments(args): - result = dirac.peekJob(job, printOutput=True) if not result["OK"]: errorList.append((job, result["Message"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py index 364f622d2ee..df7ef5bf0e2 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_submit.py @@ -48,7 +48,6 @@ def main(): jFile = open(value, "a") for jdl in args: - result = dirac.submitJob(jdl) if result["OK"]: print("JobID = %s" % (result["Value"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py index 27dccddc229..72fde4515f1 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_jobs_select_output_search.py @@ -87,11 +87,9 @@ def main(): DIRAC.exit(2) for job in jobs: - result = dirac.getOutputSandbox(job) if result["OK"]: if os.path.exists("%s" % job): - lines = [] try: lines = open(os.path.join(job, filename)).readlines() diff --git a/src/DIRAC/Interfaces/scripts/dsub.py b/src/DIRAC/Interfaces/scripts/dsub.py index 4fa88b3fd47..1d09e5ccf8c 100755 --- a/src/DIRAC/Interfaces/scripts/dsub.py +++ b/src/DIRAC/Interfaces/scripts/dsub.py @@ -52,7 +52,6 @@ def classAdAppendToOutputSandbox(classAd, f): def classAdAppendToSandbox(classAd, f, sbName): - global classAdJob sb = [] @@ -330,7 +329,6 @@ def pnumber(start, stop, step, factor): @Script() def main(): - global classAdJob params = Params() diff --git a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py index caf7fdf63c6..4146f2b94d0 100644 --- a/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py +++ b/src/DIRAC/MonitoringSystem/Client/MonitoringReporter.py @@ -42,7 +42,6 @@ class MonitoringReporter: """ def __init__(self, monitoringType="", failoverQueueName="dirac.monitoring"): - self.__maxRecordsInABundle = 5000 self.__documentLock = threading.RLock() self.__documents = [] diff --git a/src/DIRAC/MonitoringSystem/Client/Types/AgentMonitoring.py b/src/DIRAC/MonitoringSystem/Client/Types/AgentMonitoring.py index d7f0e1b12c9..f77dd747226 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/AgentMonitoring.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/AgentMonitoring.py @@ -10,7 +10,6 @@ class AgentMonitoring(BaseType): """ def __init__(self): - super().__init__() self.keyFields = [ diff --git a/src/DIRAC/MonitoringSystem/Client/Types/DataOperation.py b/src/DIRAC/MonitoringSystem/Client/Types/DataOperation.py index 3171c81d582..0871e42470d 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/DataOperation.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/DataOperation.py @@ -7,7 +7,6 @@ class DataOperation(BaseType): def __init__(self): - super().__init__() self.keyFields = [ diff --git a/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py b/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py index cc2b1175a2b..24fff2a783c 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py @@ -7,7 +7,6 @@ class FailedDataOperation(BaseType): def __init__(self): - super().__init__() self.keyFields = [ diff --git a/src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py b/src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py index 1d18e79a310..bd354654b41 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/PilotSubmissionMonitoring.py @@ -7,7 +7,6 @@ class PilotSubmissionMonitoring(BaseType): def __init__(self): - super().__init__() self.keyFields = [ diff --git a/src/DIRAC/MonitoringSystem/Client/Types/ServiceMonitoring.py b/src/DIRAC/MonitoringSystem/Client/Types/ServiceMonitoring.py index 32c789baa45..0dea9cbfb9c 100644 --- a/src/DIRAC/MonitoringSystem/Client/Types/ServiceMonitoring.py +++ b/src/DIRAC/MonitoringSystem/Client/Types/ServiceMonitoring.py @@ -10,7 +10,6 @@ class ServiceMonitoring(BaseType): """ def __init__(self): - super().__init__() self.keyFields = [ diff --git a/src/DIRAC/MonitoringSystem/DB/test/Test_monitoringdb.py b/src/DIRAC/MonitoringSystem/DB/test/Test_monitoringdb.py index 86991e583b3..c911d527b2e 100644 --- a/src/DIRAC/MonitoringSystem/DB/test/Test_monitoringdb.py +++ b/src/DIRAC/MonitoringSystem/DB/test/Test_monitoringdb.py @@ -43,7 +43,6 @@ def test_instantiate(self): ################################################################################ def test_determineBucketSize(self): - client = self.testClass("MonitoringDB", "Test") result = client._determineBucketSize(1458130176000, 1458226213000) self.assertTrue(result["OK"]) diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/BasePlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/BasePlotter.py index 7bb02c0bd87..15b47d01193 100644 --- a/src/DIRAC/MonitoringSystem/private/Plotters/BasePlotter.py +++ b/src/DIRAC/MonitoringSystem/private/Plotters/BasePlotter.py @@ -18,7 +18,6 @@ class BasePlotter(DBUtils): - _EA_THUMBNAIL = "thumbnail" _EA_WIDTH = "width" _EA_HEIGHT = "height" diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/RMSMonitoringPlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/RMSMonitoringPlotter.py index 0bf1c14b3de..f82a50e130f 100644 --- a/src/DIRAC/MonitoringSystem/private/Plotters/RMSMonitoringPlotter.py +++ b/src/DIRAC/MonitoringSystem/private/Plotters/RMSMonitoringPlotter.py @@ -23,7 +23,6 @@ class RMSMonitoringPlotter(BasePlotter): _typeKeyFields = RMSMonitoring().keyFields def __reportAllResources(self, reportRequest, metric, unit): - retVal = self._getTimedData( startTime=reportRequest["startTime"], endTime=reportRequest["endTime"], @@ -44,7 +43,6 @@ def __reportAllResources(self, reportRequest, metric, unit): return S_OK({"data": dataDict, "granularity": granularity, "unit": unitName}) def __plotAllResources(self, reportRequest, plotInfo, filename, title): - metadata = { "title": "{} by {}".format(title, reportRequest["grouping"]), "starttime": reportRequest["startTime"], diff --git a/src/DIRAC/ProductionSystem/Service/ProductionManagerHandler.py b/src/DIRAC/ProductionSystem/Service/ProductionManagerHandler.py index c6f638affd5..874eed2358a 100644 --- a/src/DIRAC/ProductionSystem/Service/ProductionManagerHandler.py +++ b/src/DIRAC/ProductionSystem/Service/ProductionManagerHandler.py @@ -123,7 +123,6 @@ def export_getProductionTransformations( limit=None, offset=None, ): - if not condDict: condDict = {} return cls.productionDB.getProductionTransformations( diff --git a/src/DIRAC/RequestManagementSystem/Client/File.py b/src/DIRAC/RequestManagementSystem/Client/File.py index 4e4d7bebbb7..5c6e35b7e00 100644 --- a/src/DIRAC/RequestManagementSystem/Client/File.py +++ b/src/DIRAC/RequestManagementSystem/Client/File.py @@ -182,7 +182,6 @@ def _getJSONData(self): jsonData = {} for attrName in attrNames: - # FileID and OperationID might not be set since they are managed by SQLAlchemy if not hasattr(self, attrName): continue diff --git a/src/DIRAC/RequestManagementSystem/Client/Operation.py b/src/DIRAC/RequestManagementSystem/Client/Operation.py index c0d5274f3d6..367a9b49d33 100644 --- a/src/DIRAC/RequestManagementSystem/Client/Operation.py +++ b/src/DIRAC/RequestManagementSystem/Client/Operation.py @@ -350,7 +350,6 @@ def _getJSONData(self): jsonData = {} for attrName in Operation.ATTRIBUTE_NAMES: - # RequestID and OperationID might not be set since they are managed by SQLAlchemy if not hasattr(self, attrName): continue diff --git a/src/DIRAC/RequestManagementSystem/Client/ReqClient.py b/src/DIRAC/RequestManagementSystem/Client/ReqClient.py index 05420e358af..ac8cfe34ac9 100755 --- a/src/DIRAC/RequestManagementSystem/Client/ReqClient.py +++ b/src/DIRAC/RequestManagementSystem/Client/ReqClient.py @@ -531,7 +531,6 @@ def printFTSJobs(request): try: if request.RequestID: - # We try first the new FTS3 system from DIRAC.DataManagementSystem.Client.FTS3Client import FTS3Client diff --git a/src/DIRAC/RequestManagementSystem/Client/Request.py b/src/DIRAC/RequestManagementSystem/Client/Request.py index 8ba7ecaa3e5..d2a6dfef92a 100644 --- a/src/DIRAC/RequestManagementSystem/Client/Request.py +++ b/src/DIRAC/RequestManagementSystem/Client/Request.py @@ -400,7 +400,6 @@ def _getJSONData(self): jsonData = {} for attrName in attrNames: - # RequestID might not be set since it is managed by SQLAlchemy if not hasattr(self, attrName): continue diff --git a/src/DIRAC/RequestManagementSystem/DB/RequestDB.py b/src/DIRAC/RequestManagementSystem/DB/RequestDB.py index 487fa11e3ad..54b90392b27 100644 --- a/src/DIRAC/RequestManagementSystem/DB/RequestDB.py +++ b/src/DIRAC/RequestManagementSystem/DB/RequestDB.py @@ -267,10 +267,8 @@ def putRequest(self, request): session = self.DBSession(expire_on_commit=False) try: - try: if hasattr(request, "RequestID"): - status = session.query(Request._Status).filter(Request.RequestID == request.RequestID).one() if status[0] == "Canceled": @@ -342,7 +340,6 @@ def getRequest(self, reqID=0, assigned=True): requestID = None try: - if reqID: requestID = reqID @@ -639,13 +636,11 @@ def getRequestSummaryWeb(self, selectDict, sortList, startItem, maxItems): ) for key, value in selectDict.items(): - if key == "ToDate": summaryQuery = summaryQuery.filter(Request._LastUpdate < value) elif key == "FromDate": summaryQuery = summaryQuery.filter(Request._LastUpdate > value) else: - tableName = "Request" if key == "Type": @@ -742,7 +737,6 @@ def getRequestCountersWeb(self, groupingAttribute, selectDict): elif key == "FromDate": summaryQuery = summaryQuery.filter(Request._LastUpdate > value) else: - objectType = "Request" if key == "Type": summaryQuery = summaryQuery.join(Request.__operations__) @@ -925,7 +919,6 @@ def getRequestInfo(self, requestID): session = self.DBSession() try: - requestInfoQuery = session.query( Request.RequestID, Request._Status, diff --git a/src/DIRAC/RequestManagementSystem/private/JSONUtils.py b/src/DIRAC/RequestManagementSystem/private/JSONUtils.py index 462b2b5b0b8..7fabcbb116e 100644 --- a/src/DIRAC/RequestManagementSystem/private/JSONUtils.py +++ b/src/DIRAC/RequestManagementSystem/private/JSONUtils.py @@ -5,7 +5,6 @@ class RMSEncoder(json.JSONEncoder): """This class is an encoder for the Requests, Operation and Files.""" def default(self, obj): # pylint: disable=method-hidden - if hasattr(obj, "_getJSONData"): return obj._getJSONData() elif isinstance(obj, bytes): diff --git a/src/DIRAC/RequestManagementSystem/private/RequestTask.py b/src/DIRAC/RequestManagementSystem/private/RequestTask.py index db214da33d4..6981e333128 100644 --- a/src/DIRAC/RequestManagementSystem/private/RequestTask.py +++ b/src/DIRAC/RequestManagementSystem/private/RequestTask.py @@ -270,7 +270,6 @@ def __call__(self): error = None while self.request.Status == "Waiting": - # # get waiting operation operation = self.request.getWaiting() if not operation["OK"]: diff --git a/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py index 8c74ffeed8d..876668dae4b 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/CacheFeederAgent.py @@ -28,7 +28,6 @@ class CacheFeederAgent(AgentModule): """ def __init__(self, *args, **kwargs): - AgentModule.__init__(self, *args, **kwargs) self.commands = {} @@ -123,11 +122,9 @@ def execute(self): """Just executes, via `loadCommand`, the commands in self.commands one after the other""" for commandModule, commandList in self.commands.items(): - self.log.info("%s module initialization" % commandModule) for commandDict in commandList: - commandObject = self.loadCommand(commandModule, commandDict) if not commandObject["OK"]: self.log.error(commandObject["Message"]) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py index cf9c67e2365..08e5b69edfd 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/ElementInspectorAgent.py @@ -98,7 +98,6 @@ def execute(self): # filter elements by Type for element in res["Value"]: - # Maybe an overkill, but this way I have NEVER again to worry about order # of elements returned by mySQL on tuples elemDict = dict(zip(res["Columns"], element)) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/EmailAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/EmailAgent.py index 561868e0936..4dd568b8ccf 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/EmailAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/EmailAgent.py @@ -20,7 +20,6 @@ class EmailAgent(AgentModule): def __init__(self, *args, **kwargs): - AgentModule.__init__(self, *args, **kwargs) self.diracAdmin = None self.default_value = None @@ -53,7 +52,6 @@ def _groupBySiteName(result): return siteNameDict def execute(self): - result = self.rsClient.select("ResourceStatusCache") if not result["OK"]: return S_ERROR() @@ -62,7 +60,6 @@ def execute(self): result = self._groupBySiteName(result) for site, records in result.items(): - email = "" html_body = "" html_elements = "" @@ -149,7 +146,6 @@ def execute(self): return S_OK() def _sendMail(self, subject, body, html=False): - userEmails = self._getUserEmails() if not userEmails["OK"]: return userEmails @@ -158,7 +154,6 @@ def _sendMail(self, subject, body, html=False): fromAddress = RssConfiguration.RssConfiguration().getConfigFromAddress() for user in userEmails["Value"]: - # FIXME: should not I get the info from the RSS User cache ? resEmail = self.diracAdmin.sendMail(user, subject, body, fromAddress=fromAddress, html=html) @@ -168,7 +163,6 @@ def _sendMail(self, subject, body, html=False): return S_OK() def _getUserEmails(self): - configResult = RssConfiguration.getnotificationGroups() if not configResult["OK"]: return configResult diff --git a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py index f0711a817fd..227b118fa96 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/SiteInspectorAgent.py @@ -37,7 +37,6 @@ class SiteInspectorAgent(AgentModule): __checkingFreqs = {"Active": 20, "Degraded": 20, "Probing": 20, "Banned": 15, "Unknown": 10, "Error": 5} def __init__(self, *args, **kwargs): - AgentModule.__init__(self, *args, **kwargs) self.rsClient = None @@ -83,7 +82,6 @@ def execute(self): # filter elements for site in res["Value"]: - # Maybe an overkill, but this way I have NEVER again to worry about order # of elements returned by mySQL on tuples siteDict = dict(zip(res["Columns"], site)) diff --git a/src/DIRAC/ResourceStatusSystem/Agent/SummarizeLogsAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/SummarizeLogsAgent.py index c06a00223cd..7824300059f 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/SummarizeLogsAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/SummarizeLogsAgent.py @@ -64,7 +64,6 @@ def execute(self): # loop over the tables for element in ("Site", "Resource", "Node"): - self.log.info("Summarizing %s" % element) # get all logs to be summarized @@ -78,7 +77,6 @@ def execute(self): # logElements is a dictionary of key-value pairs as follows: # (name, statusType) : list(logs) for key, logs in logElements.items(): - sumResult = self._registerLogs(element, key, logs) if not sumResult["OK"]: self.log.error(sumResult["Message"]) @@ -121,7 +119,6 @@ def _summarizeLogs(self, element): latestID = dict(zip(selectColumns, selectResults[-1]))["ID"] for selectResult in selectResults: - elementDict = dict(zip(selectColumns, selectResult)) key = (elementDict["Name"], elementDict["StatusType"]) @@ -197,7 +194,6 @@ def _registerLogs(self, element, key, logs): self.log.debug(logs) for selectedItemDict in logs: - res = self.__logToHistoryTable(element, selectedItemDict) if not res["OK"]: return res diff --git a/src/DIRAC/ResourceStatusSystem/Agent/TokenAgent.py b/src/DIRAC/ResourceStatusSystem/Agent/TokenAgent.py index 0287372312b..e8613db70bd 100644 --- a/src/DIRAC/ResourceStatusSystem/Agent/TokenAgent.py +++ b/src/DIRAC/ResourceStatusSystem/Agent/TokenAgent.py @@ -63,7 +63,6 @@ def execute(self): elements = ("Site", "Resource", "Node") for element in elements: - self.log.info("Processing %s" % element) interestingTokens = self._getInterestingTokens(element) @@ -104,7 +103,6 @@ def _getInterestingTokens(self, element): interestingTokens = [] for tokenElement in tokenElements: - tokenElement = dict(zip(tokenColumns, tokenElement)) if tokenElement["TokenOwner"] != self.__rssToken: @@ -121,7 +119,6 @@ def _processTokens(self, element, tokenElements): never = datetime.max for tokenElement in tokenElements: - try: name = tokenElement["Name"] statusType = tokenElement["StatusType"] @@ -170,12 +167,10 @@ def _notifyOfTokens(self): adminExpiring = [] for tokenOwner, tokenLists in self.tokenDict.items(): - expired = [] expiring = [] for tokenList in tokenLists: - if tokenList[5] < now: expired.append(tokenList) adminExpired.append(tokenList) @@ -202,13 +197,11 @@ def _notify(self, tokenOwner, expired, expiring): mail = "\nEXPIRED tokens ( RSS has taken control of them )\n" for tokenList in expired: - mail += " ".join([str(x) for x in tokenList]) mail += "\n" mail = "\nEXPIRING tokens ( RSS will take control of them )\n" for tokenList in expiring: - mail += " ".join([str(x) for x in tokenList]) mail += "\n" diff --git a/src/DIRAC/ResourceStatusSystem/Client/PublisherClient.py b/src/DIRAC/ResourceStatusSystem/Client/PublisherClient.py index db46e50b795..aadf1b07b39 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/PublisherClient.py +++ b/src/DIRAC/ResourceStatusSystem/Client/PublisherClient.py @@ -5,6 +5,5 @@ class PublisherClient(Client): def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("ResourceStatus/Publisher") diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py index d6e441426bb..45029674ce2 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceManagementClient.py @@ -38,7 +38,6 @@ class ResourceManagementClient(Client): """ def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("ResourceStatus/ResourceManagement") diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py index fae16a4dfcd..178e717da2d 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py @@ -217,7 +217,6 @@ def __getCSElementStatus(self, elementName, elementType, statusType, default): result = {} for element in elementName: - for sType in statusType: # Look in standard location, 'Active' by default res = gConfig.getValue(f"{cs_path}/{element}/{sType}", "Active") diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatusClient.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatusClient.py index e2beac9f944..43aa90bd854 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatusClient.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatusClient.py @@ -25,7 +25,6 @@ class ResourceStatusClient(Client): """ def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("ResourceStatus/ResourceStatus") @@ -572,7 +571,6 @@ def __extermineStatusElement(self, element, name, keepLogs): tableTypes.append("Log") for table in tableTypes: - deleteQuery = self.deleteStatusElement(element, table, name=name) if not deleteQuery["OK"]: return deleteQuery diff --git a/src/DIRAC/ResourceStatusSystem/Command/AccountingCacheCommand.py b/src/DIRAC/ResourceStatusSystem/Command/AccountingCacheCommand.py index 20b5e54d664..a0224a42c1f 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/AccountingCacheCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/AccountingCacheCommand.py @@ -19,7 +19,6 @@ class SuccessfullJobsBySiteSplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -90,7 +89,6 @@ def doCommand(self): class FailedJobsBySiteSplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -166,7 +164,6 @@ def doCommand(self): class SuccessfullPilotsBySiteSplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -242,7 +239,6 @@ def doCommand(self): class FailedPilotsBySiteSplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -318,7 +314,6 @@ def doCommand(self): class SuccessfullPilotsByCESplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -389,7 +384,6 @@ def doCommand(self): class FailedPilotsByCESplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -467,7 +461,6 @@ def doCommand(self): class RunningJobsBySiteSplittedCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: diff --git a/src/DIRAC/ResourceStatusSystem/Command/Command.py b/src/DIRAC/ResourceStatusSystem/Command/Command.py index 9ebe99479f6..e967f84021e 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/Command.py +++ b/src/DIRAC/ResourceStatusSystem/Command/Command.py @@ -11,7 +11,6 @@ class Command: """ def __init__(self, args=None, clients=None): - self.apis = (1 and clients) or {} self.masterMode = False self.onlyCache = False diff --git a/src/DIRAC/ResourceStatusSystem/Command/DIRACAccountingCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DIRACAccountingCommand.py index 9e777ba12eb..d56d2af27c6 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DIRACAccountingCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DIRACAccountingCommand.py @@ -21,7 +21,6 @@ class DIRACAccountingCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -89,7 +88,6 @@ def doCommand(self): class TransferQualityCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -145,7 +143,6 @@ def doCommand(self): # FIXME: WHAT the hell is this doing ? values = [] if len(pr_q_d) == 1: - for k in pr_q_d: for n in pr_q_d[k].values(): values.append(n) @@ -202,7 +199,6 @@ def doCommand(self): class CachedPlotCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ResourceManagementClient" in self.apis: @@ -270,7 +266,6 @@ def doCommand(self): class TransferQualityFromCachedPlotCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ResourceManagementClient" in self.apis: diff --git a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py index 7bb6677fe09..e511b3ca1f3 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py @@ -30,7 +30,6 @@ class DowntimeCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "GOCDBClient" in self.apis: @@ -130,7 +129,6 @@ def _prepareCommand(self): # Transform DIRAC site names into GOCDB topics if element == "Site": - gocSite = getGOCSiteName(elementName) if not gocSite["OK"]: # The site is most probably is not a grid site - not an issue, of course pass # so, elementName remains unchanged @@ -246,7 +244,6 @@ def doNew(self, masterParams=None): # Humanize the results into a dictionary, not the most optimal, but readable for downtime, downDic in results.items(): # can be an iterator - dt = {} dt["Name"] = downDic.get("URL", downDic.get("HOSTNAME", downDic.get("SITENAME"))) diff --git a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 38a1f4108d4..fe1b456f42c 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -29,7 +29,6 @@ class FreeDiskSpaceCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients=clients) self.rmClient = ResourceManagementClient() diff --git a/src/DIRAC/ResourceStatusSystem/Command/GGUSTicketsCommand.py b/src/DIRAC/ResourceStatusSystem/Command/GGUSTicketsCommand.py index 3ec0e6ee500..97e75b39452 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/GGUSTicketsCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/GGUSTicketsCommand.py @@ -19,7 +19,6 @@ class GGUSTicketsCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "GGUSTicketsClient" in self.apis: @@ -94,7 +93,6 @@ def doNew(self, masterParams=None): uniformResult = [] for gocSite, ggusResult in results.items(): - if gocSite not in gocNames: continue @@ -160,7 +158,6 @@ def doMaster(self): self.log.info("Processing %s" % ", ".join(gocSites)) for gocNameToQuery in gocSites: - # if gocNameToQuery is None: # self.metrics[ 'failed' ].append( 'None result' ) # continue diff --git a/src/DIRAC/ResourceStatusSystem/Command/GOCDBSyncCommand.py b/src/DIRAC/ResourceStatusSystem/Command/GOCDBSyncCommand.py index a230f2d54d2..bd607c4856c 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/GOCDBSyncCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/GOCDBSyncCommand.py @@ -17,7 +17,6 @@ class GOCDBSyncCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "GOCDBClient" in self.apis: @@ -51,7 +50,6 @@ def doNew(self, masterParams=None): return result for downtimes in result["Value"]: - localDBdict = { "DowntimeID": downtimes[3], "FORMATED_START_DATE": downtimes[6].strftime("%Y-%m-%d %H:%M"), @@ -75,7 +73,6 @@ def doNew(self, masterParams=None): GOCDBDowntimeID = GOCDBdict["PRIMARY_KEY"] + " " + GOCDBdict["ENDPOINT"] if localDowntimeID == GOCDBDowntimeID: - if localDBdict["FORMATED_START_DATE"] != GOCDBdict["FORMATED_START_DATE"]: result = self.rmClient.addOrModifyDowntimeCache( downtimeID=localDBdict["DowntimeID"], startDate=GOCDBdict["FORMATED_START_DATE"] @@ -113,7 +110,6 @@ def doMaster(self): return result for data in result["Value"]: - # If already processed don't do it again if data[0] in self.seenHostnames: continue diff --git a/src/DIRAC/ResourceStatusSystem/Command/JobCommand.py b/src/DIRAC/ResourceStatusSystem/Command/JobCommand.py index 6ba46948cd5..3de7d4991b1 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/JobCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/JobCommand.py @@ -16,7 +16,6 @@ class JobCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "WMSAdministrator" in self.apis: @@ -35,7 +34,6 @@ def _storeCommand(self, result): """ for jobDict in result: - resQuery = self.rmClient.addOrModifyJobCache( jobDict["Site"], jobDict["MaskStatus"], jobDict["Efficiency"], jobDict["Status"] ) @@ -92,7 +90,6 @@ def doNew(self, masterParams=None): uniformResult = [] for record in records: - # This returns a dictionary with the following keys # 'Site', 'GridType', 'Country', 'Tier', 'MaskStatus', 'Received', # 'Checking', 'Staging', 'Waiting', 'Matched', 'Running', 'Stalled', @@ -148,7 +145,6 @@ def doMaster(self): class JobsWMSCommand(Command): def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "WMSAdministrator" in self.apis: @@ -196,7 +192,6 @@ def doCommand(self): jobResults = [] for record in records: - jobDict = dict(zip(params, record)) try: jobDict["Efficiency"] = float(jobDict["Efficiency"]) diff --git a/src/DIRAC/ResourceStatusSystem/Command/MacroCommand.py b/src/DIRAC/ResourceStatusSystem/Command/MacroCommand.py index a7f27e49423..f15e2c68bd0 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/MacroCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/MacroCommand.py @@ -14,7 +14,6 @@ class MacroCommand(Command): """ def __init__(self): - super().__init__() self.commands = None diff --git a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py index 3bbf0518422..73661062490 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/PilotCommand.py @@ -17,7 +17,6 @@ class PilotCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "Pilots" in self.apis: @@ -36,7 +35,6 @@ def _storeCommand(self, result): """ for pilotDict in result: - resQuery = self.rmClient.addOrModifyPilotCache( site=pilotDict["Site"], cE=pilotDict["CE"], @@ -75,7 +73,6 @@ def _prepareCommand(self): return S_OK((element, name, vo)) def doNew(self, masterParams=None): - self.log.debug("PilotCommand doNew") if masterParams is not None: element, name = masterParams @@ -118,7 +115,6 @@ def doNew(self, masterParams=None): uniformResult = [] for record in records: - # This returns a dictionary with the following keys: # 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running', # 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob', @@ -137,7 +133,6 @@ def doNew(self, masterParams=None): return S_OK(uniformResult) def doCache(self): - self.log.debug("PilotCommand doCache") params = self._prepareCommand() if not params["OK"]: @@ -160,7 +155,6 @@ def doCache(self): return result def doMaster(self): - self.log.debug("PilotCommand doMaster") siteNames = getSites() if not siteNames["OK"]: diff --git a/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py b/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py index f66a10977dc..0fd578160d5 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/PropagationCommand.py @@ -12,7 +12,6 @@ class PropagationCommand(Command): def __init__(self, args=None, clients=None): - self.rssClient = ResourceStatusClient() super().__init__(args, clients) @@ -20,7 +19,6 @@ def doNew(self, masterParams=None): return S_OK() def doCache(self): - if not self.args["name"]: return S_ERROR("site was not found in args") diff --git a/src/DIRAC/ResourceStatusSystem/Command/TransferCommand.py b/src/DIRAC/ResourceStatusSystem/Command/TransferCommand.py index 3091cbeace6..8bc41e196e2 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/TransferCommand.py +++ b/src/DIRAC/ResourceStatusSystem/Command/TransferCommand.py @@ -17,7 +17,6 @@ class TransferCommand(Command): """ def __init__(self, args=None, clients=None): - super().__init__(args, clients) if "ReportsClient" in self.apis: @@ -36,7 +35,6 @@ def _storeCommand(self, results): """ for result in results: - resQuery = self.rmClient.addOrModifyTransferCache( result["SourceName"], result["DestinationName"], result["Metric"], result["Value"] ) @@ -123,7 +121,6 @@ def doNew(self, masterParams=None): uniformResult = [] for channel, elementDict in transferResults.items(): - try: source, destination = channel.split(" -> ") except ValueError: diff --git a/src/DIRAC/ResourceStatusSystem/Policy/PropagationPolicy.py b/src/DIRAC/ResourceStatusSystem/Policy/PropagationPolicy.py index bea819e73cf..414dc96bab5 100644 --- a/src/DIRAC/ResourceStatusSystem/Policy/PropagationPolicy.py +++ b/src/DIRAC/ResourceStatusSystem/Policy/PropagationPolicy.py @@ -37,13 +37,11 @@ def _evaluate(commandResult): result = {"Status": None, "Reason": None} if not commandResult["OK"]: - result["Status"] = "Error" result["Reason"] = commandResult["Message"] return S_OK(result) else: - commandResult = commandResult["Value"] result["Status"] = commandResult["Status"] diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/BaseAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/BaseAction.py index 9206590f6e4..d4baff51ac1 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/BaseAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/BaseAction.py @@ -12,7 +12,6 @@ class BaseAction: """ def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients): - # enforcementResult supposed to look like: # { # 'Status' : , diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/EmailAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/EmailAction.py index 4db9683fcd2..2a4a11a4c78 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/EmailAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/EmailAction.py @@ -13,7 +13,6 @@ class EmailAction(BaseAction): def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients=None): - super().__init__(name, decisionParams, enforcementResult, singlePolicyResults, clients) if clients is not None and "ResourceStatusClient" in clients: diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py index 20ee4f36998..2dbf3ab3c04 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py @@ -17,7 +17,6 @@ class LogPolicyResultAction(BaseAction): """ def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients=None): - super().__init__(name, decisionParams, enforcementResult, singlePolicyResults, clients) if clients is not None and "ResourceManagementClient" in clients: @@ -44,7 +43,6 @@ def run(self): return S_ERROR("statusType should not be None") for singlePolicyResult in self.singlePolicyResults: - status = singlePolicyResult["Status"] if status is None: return S_ERROR("status should not be None") diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogStatusAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogStatusAction.py index 66892378c20..6a6443bbf8b 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogStatusAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogStatusAction.py @@ -13,7 +13,6 @@ class LogStatusAction(BaseAction): """ def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients=None): - super().__init__(name, decisionParams, enforcementResult, singlePolicyResults, clients) if clients is not None and "ResourceStatusClient" in clients: diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/SlackAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/SlackAction.py index 09388afddda..a71707246c2 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/SlackAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/SlackAction.py @@ -39,7 +39,6 @@ class SlackAction(BaseAction): """ def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, clients=None, url=None): - super().__init__(name, decisionParams, enforcementResult, singlePolicyResults, clients) if url is not None: self.url = url diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/PDP.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/PDP.py index 20c8ced8537..fbe0a0f5311 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/PDP.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/PDP.py @@ -207,7 +207,6 @@ def _runPolicies(self, policies): validStatus = self.rssMachine.getStates() for policyDict in policies: - # Load and evaluate policy described in for element described # in policyInvocationResult = self.pCaller.policyInvocation(self.decisionParams, policyDict) @@ -308,14 +307,12 @@ def _combineSinglePolicyResults(self, singlePolicyRes): # If the RssMachine does not accept the candidate, return forcing message if candidateState != nextState: - policyCombined["Status"] = nextState policyCombined["Reason"] = f"RssMachine forced status {candidateState} to {nextState}" return S_OK(policyCombined) # If the RssMachine accepts the candidate, just concatenate the reasons for policyRes in policiesToCombine: - if policyRes["Status"] == nextState: policyCombined["Reason"] += "%s ###" % policyRes["Reason"] diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py index cff8206e670..7f7ab7d4529 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py @@ -140,7 +140,6 @@ def enforce(self, decisionParams): return isNotUpdated for policyActionName, policyActionType in policyCombinedResult["PolicyAction"]: - try: actionMod = Utils.voimport("DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" % policyActionType) except ImportError: diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/test/Test_PolicySystem.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/test/Test_PolicySystem.py index d7c58fc205e..58690e0ad85 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/test/Test_PolicySystem.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/test/Test_PolicySystem.py @@ -29,11 +29,9 @@ def setUp(self): class PEPSuccess(PolicySystemTestCase): - ############################################################################# def test_enforce(self): - pep = PEP( {"ResourceStatusClient": self.RSMock, "ResourceManagementClient": self.RMMock, "SiteStatus": self.RMMock} ) diff --git a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py index 11052725180..416b9f2de05 100644 --- a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py +++ b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py @@ -68,7 +68,6 @@ def export_getSitesResources(self, siteNames): sitesRes = {} for siteName in siteNames: - result = getSiteCEMapping() if not result["OK"]: self.log.error("Error getting sites/CEs mapping", result["Message"]) @@ -202,7 +201,6 @@ def export_getTree(self, elementType, elementName): return sesStatus def feedTree(elementsList): - elements = {} for elementTuple in elementsList["Value"]: name, statusType, status = elementTuple @@ -221,7 +219,6 @@ def feedTree(elementsList): types_setToken = [str] * 7 def export_setToken(self, element, name, statusType, token, elementType, username, lastCheckTime): - lastCheckTime = datetime.strptime(lastCheckTime, "%Y-%m-%d %H:%M:%S") credentials = self.getRemoteCredentials() @@ -277,13 +274,11 @@ def getSite(self, elementType, elementName): domainNames = result["Value"] for domainName in domainNames: - sites = gConfig.getSections("Resources/Sites/%s" % domainName) if not sites["OK"]: continue for site in sites["Value"]: - elements = gConfig.getValue(f"Resources/Sites/{domainName}/{site}/{elementType}", "") if elementName in elements: return S_OK(site) @@ -296,7 +291,6 @@ def getSite(self, elementType, elementName): @classmethod def export_getDowntimes(cls, element, elementType, name): - if elementType == "StorageElement": res = getSEHosts(name) if not res["OK"]: @@ -317,7 +311,6 @@ def export_getDowntimes(cls, element, elementType, name): ] def export_getCachedDowntimes(self, element, elementType, name, severity): - if elementType == "StorageElement": res = getSEHosts(name) if not res["OK"]: @@ -343,7 +336,6 @@ def export_getCachedDowntimes(self, element, elementType, name, severity): types_setStatus = [str] * 7 def export_setStatus(self, element, name, statusType, status, elementType, username, lastCheckTime): - if not lastCheckTime: lastCheckTime = None else: diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py b/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py index aa8ddc53bbc..a6f4ebc691b 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py @@ -43,7 +43,6 @@ def getPoliciesThatApply(decisionParams): # Get policies that match the given decisionParameters for policyName, policySetup in policiesConfig.items(): - # The parameter policyType replaces policyName, so if it is not present, # we pick policyName try: @@ -75,7 +74,6 @@ def getPoliciesThatApply(decisionParams): policiesToBeLoaded = [] # Gets policies parameters from code. for policyName, policyType, _policyConfigParams in policiesThatApply: - try: configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations") policies = copy.deepcopy(configModule.POLICIESMETA) @@ -124,7 +122,6 @@ def getPolicyActionsThatApply(decisionParams, singlePolicyResults, policyCombine # Get policies that match the given decissionParameters for policyActionName, policyActionConfig in policyActionsConfig.items(): - # The parameter policyType is mandatory. If not present, we pick policyActionName try: policyActionType = policyActionConfig["actionType"][0] diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/RSSCache.py b/src/DIRAC/ResourceStatusSystem/Utilities/RSSCache.py index c77606f9927..90d4d30b6d8 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/RSSCache.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/RSSCache.py @@ -140,7 +140,6 @@ def getBulk(self, resourceKeys): self.__rssCacheLock.acquire() for resourceKey in resourceKeys: - resourceRow = self.__rssCache.get(resourceKey) if not resourceRow: return S_ERROR("Cannot get %s" % resourceKey) @@ -222,7 +221,6 @@ def __refreshCacheThreadRun(self): """ while not self.__refreshStop: - self.__rssCacheLock.acquire() self.refreshCacheAndHistory() self.__rssCacheLock.release() diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/RssConfiguration.py b/src/DIRAC/ResourceStatusSystem/Utilities/RssConfiguration.py index 49bc5e8d983..1e4e8e10a7b 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/RssConfiguration.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/RssConfiguration.py @@ -68,7 +68,6 @@ def getConfigStatusType(self, elementType=None): res = self.opsHelper.getOptionsDict("%s/Config/StatusTypes" % _rssConfigPath) if res["OK"]: - if elementType in res["Value"]: return List.fromChar(res["Value"][elementType]) diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py b/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py index 874588368a6..e8a1175f220 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py @@ -31,7 +31,6 @@ class Synchronizer: """ def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): - # Warm up local CS CSHelpers.warmUp() @@ -252,7 +251,6 @@ def __syncComputingElements(self): # Delete storage elements for ceName in toBeDeleted: - deleteQuery = self.rStatus._extermineStatusElement("Resource", ceName) gLogger.verbose("... %s" % ceName) @@ -276,7 +274,6 @@ def __syncComputingElements(self): gLogger.debug("%s Computing elements entries to be added" % len(toBeAdded)) for ceTuple in toBeAdded: - _name = ceTuple[0] _statusType = ceTuple[1] _status = self.defaultStatus @@ -323,7 +320,6 @@ def __syncFileCatalogs(self): # Delete storage elements for catalogName in toBeDeleted: - deleteQuery = self.rStatus._extermineStatusElement("Resource", catalogName) gLogger.verbose("... %s" % catalogName) @@ -347,7 +343,6 @@ def __syncFileCatalogs(self): gLogger.verbose("%s File catalogs entries to be added" % len(toBeAdded)) for catalogTuple in toBeAdded: - _name = catalogTuple[0] _statusType = catalogTuple[1] _status = self.defaultStatus @@ -392,7 +387,6 @@ def __syncFTS(self): # Delete storage elements for ftsName in toBeDeleted: - deleteQuery = self.rStatus._extermineStatusElement("Resource", ftsName) gLogger.verbose("... %s" % ftsName) @@ -416,7 +410,6 @@ def __syncFTS(self): gLogger.verbose("%s FTS endpoints entries to be added" % len(toBeAdded)) for ftsTuple in toBeAdded: - _name = ftsTuple[0] _statusType = ftsTuple[1] _status = self.defaultStatus @@ -460,7 +453,6 @@ def __syncStorageElements(self): # Delete storage elements for sesName in toBeDeleted: - deleteQuery = self.rStatus._extermineStatusElement("Resource", sesName) gLogger.verbose("... %s" % sesName) @@ -484,7 +476,6 @@ def __syncStorageElements(self): gLogger.verbose("%s storage element entries to be added" % len(toBeAdded)) for seTuple in toBeAdded: - _name = seTuple[0] _statusType = seTuple[1] _status = self.defaultStatus @@ -529,7 +520,6 @@ def __syncQueues(self): # Delete storage elements for queueName in toBeDeleted: - deleteQuery = self.rStatus._extermineStatusElement("Node", queueName) gLogger.verbose("... %s" % queueName) @@ -553,7 +543,6 @@ def __syncQueues(self): gLogger.verbose("%s Queue entries to be added" % len(toBeAdded)) for queueTuple in toBeAdded: - _name = queueTuple[0] _statusType = queueTuple[1] _status = self.defaultStatus diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py b/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py index cfbe805a1bd..90ea95d1f33 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py @@ -17,7 +17,6 @@ def voimport(base_mod): """ for ext in gConfig.getValue("DIRAC/Extensions", []): - try: return __import__(ext + base_mod, globals(), locals(), ["*"]) except ImportError: @@ -44,7 +43,6 @@ def getCSTreeAsDict(treePath): opts = opHelper.getOptionsDict(treePath) if opts["OK"]: - opts = opts["Value"] for optKey, optValue in opts.items(): @@ -56,11 +54,9 @@ def getCSTreeAsDict(treePath): secs = opHelper.getSections(treePath) if secs["OK"]: - secs = secs["Value"] for sec in secs: - secTree = getCSTreeAsDict(f"{treePath}/{sec}") if not secTree["OK"]: return secTree @@ -83,7 +79,6 @@ def configMatch(candidateParams, configParams): """ for key in candidateParams: - if key not in configParams: # The candidateParams is missing one of the parameters required # return False diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py index 8abb97b0800..e7720523f66 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_list_status.py @@ -107,7 +107,6 @@ def tabularPrint(elementsList): gLogger.notice(" {}: {}".format("element".ljust(15), switchDict["element"])) titles = [] for key in ("Name", "StatusType", "Status", "ElementType", "TokenOwner"): - # Transforms from upper lower case to lower upper case keyT = key[0].lower() + key[1:] diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py index 93761bde0a0..192ed259b63 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_db.py @@ -233,7 +233,6 @@ def confirm(query, matches): def tabularPrint(table): - columns_names = list(table[0]) records = [] for row in table: diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py index e8997b7ebe3..083cbe3562b 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py @@ -180,7 +180,6 @@ def confirm(query, matches): def tabularPrint(table): - columns_names = list(table[0]) records = [] for row in table: diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py index 9ddabc75a6c..19f8976a703 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_status.py @@ -52,7 +52,6 @@ def parseSwitches(): switches.setdefault("VO", None) for key in ("element", "name", "status", "reason"): - if key not in switches: gLogger.error("%s Switch missing" % key) gLogger.error("Please, check documentation below") @@ -184,7 +183,6 @@ def setStatus(switchDict, tokenOwner): tomorrow = datetime.utcnow().replace(microsecond=0) + timedelta(days=1) for status, statusType in elements: - gLogger.debug(f"{status} {statusType}") if switchDict["status"] == status: diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py index 8db407508f7..939ac839d59 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_set_token.py @@ -62,7 +62,6 @@ def parseSwitches(): switches["releaseToken"] = False for key in ("element", "name", "reason"): - if key not in switches: gLogger.error("%s Switch missing" % key) gLogger.error("Please, check documentation above") @@ -136,7 +135,6 @@ def setToken(user): gLogger.always(f"New token: {newTokenOwner} --- until {tokenExpiration}") for statusType, tokenOwner in elements: - # If a user different than the one issuing the command and RSS if tokenOwner != user and tokenOwner != "rs_svc": gLogger.info("{}({}) belongs to the user: {}".format(switchDict["name"], statusType, tokenOwner)) diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py index 3bc32e6e834..e2d6b8dac04 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_sync.py @@ -146,7 +146,6 @@ def initSEs(): gLogger.debug(statusTypes) for se in DMSHelpers().getStorageElements(): - gLogger.debug(se) opts = gConfig.getOptionsDict("/Resources/StorageElements/%s" % se) @@ -161,7 +160,6 @@ def initSEs(): statusTypesList = statusTypes[:] for statusType, status in opts.items(): - # Sanity check... if statusType not in statusTypesList: continue @@ -196,7 +194,6 @@ def initSEs(): # Backtracking: statusTypes not present on CS for statusType in statusTypesList: - result = rssClient.addOrModifyStatusElement( "Resource", "Status", @@ -225,7 +222,6 @@ def run(): DIRACExit(1) if "init" in switchDict: - if switchDict.get("element") == "Site": result = initSites() if not result["OK"]: diff --git a/src/DIRAC/Resources/Catalog/FileCatalog.py b/src/DIRAC/Resources/Catalog/FileCatalog.py index d8bce528f6c..c6b3c4b1dc7 100644 --- a/src/DIRAC/Resources/Catalog/FileCatalog.py +++ b/src/DIRAC/Resources/Catalog/FileCatalog.py @@ -199,7 +199,6 @@ def w_execute(self, *parms, **kws): parms1 = parms[1:] for catalogName, oCatalog, master in self.writeCatalogs: - # Skip if the method is not implemented in this catalog # NOTE: it is impossible for the master since the write method list is populated # only from the master catalog, and if the method is not there, __getattr__ @@ -296,7 +295,6 @@ def r_execute(self, *parms, **kws): successful = {} failed = {} for _catalogName, oCatalog, _master in self.readCatalogs: - # Skip if the method is not implemented in this catalog if not oCatalog.hasCatalogMethod(self.call): continue diff --git a/src/DIRAC/Resources/Catalog/FileCatalogClient.py b/src/DIRAC/Resources/Catalog/FileCatalogClient.py index ac476c675ae..ca97a01aef5 100644 --- a/src/DIRAC/Resources/Catalog/FileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/FileCatalogClient.py @@ -190,7 +190,6 @@ def setReplicaProblematic(self, lfns, revert=False): lfnsSEs = {lfn: [se for se in lfns[lfn]] for lfn in lfns} while lfnsSEs: - # { lfn : { 'SE' : se1, 'PFN' : pfn1, 'Status' : status }, ... } batch = {} diff --git a/src/DIRAC/Resources/Catalog/FileCatalogFactory.py b/src/DIRAC/Resources/Catalog/FileCatalogFactory.py index ce0b6d5b513..409e6f1c365 100644 --- a/src/DIRAC/Resources/Catalog/FileCatalogFactory.py +++ b/src/DIRAC/Resources/Catalog/FileCatalogFactory.py @@ -43,7 +43,6 @@ def __getCatalogClass(self, catalogType): return result def __createCatalog(self, catalogName, catalogType, catalogURL, optionsDict): - self.log.debug(f"Creating {catalogName} client of type {catalogType}") result = self.__getCatalogClass(catalogType) diff --git a/src/DIRAC/Resources/Catalog/PoolXMLCatalog.py b/src/DIRAC/Resources/Catalog/PoolXMLCatalog.py index f549bcdc8f2..df26dad32a5 100755 --- a/src/DIRAC/Resources/Catalog/PoolXMLCatalog.py +++ b/src/DIRAC/Resources/Catalog/PoolXMLCatalog.py @@ -16,7 +16,6 @@ class PoolFile: """ def __init__(self, dom=None): - self.guid = "" self.pfns = [] self.lfns = [] diff --git a/src/DIRAC/Resources/Catalog/PoolXMLFile.py b/src/DIRAC/Resources/Catalog/PoolXMLFile.py index deb82de0ef3..3f7c619c32f 100644 --- a/src/DIRAC/Resources/Catalog/PoolXMLFile.py +++ b/src/DIRAC/Resources/Catalog/PoolXMLFile.py @@ -104,7 +104,6 @@ def getType(fileNames, directory=""): def _getPoolCatalogs(directory=""): - patterns = ["*.xml", "*.xml*gz"] omissions = [r"\.bak$"] # to be ignored for production files diff --git a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py index 75eb5a6d876..42a079cca2b 100644 --- a/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/RucioFileCatalogClient.py @@ -43,7 +43,6 @@ def get_scope(lfn, scopes=None, diracAlgorithm="dirac"): class RucioFileCatalogClient(FileCatalogClientBase): - READ_METHODS = FileCatalogClientBase.READ_METHODS + [ "isLink", "readLink", diff --git a/src/DIRAC/Resources/Catalog/TSCatalogClient.py b/src/DIRAC/Resources/Catalog/TSCatalogClient.py index 9bf08335840..bb74cba2890 100644 --- a/src/DIRAC/Resources/Catalog/TSCatalogClient.py +++ b/src/DIRAC/Resources/Catalog/TSCatalogClient.py @@ -17,7 +17,6 @@ class TSCatalogClient(FileCatalogClientBase): NO_LFN_METHODS = ["setMetadata"] def __init__(self, url=None, **kwargs): - self.serverURL = "Transformation/TransformationManager" if not url else url super().__init__(self.serverURL, **kwargs) diff --git a/src/DIRAC/Resources/Catalog/Utilities.py b/src/DIRAC/Resources/Catalog/Utilities.py index ca796a766af..72004881c8e 100644 --- a/src/DIRAC/Resources/Catalog/Utilities.py +++ b/src/DIRAC/Resources/Catalog/Utilities.py @@ -65,7 +65,6 @@ def checkCatalogArguments(f): @functools.wraps(f) def processWithCheckingArguments(*args, **kwargs): - checkFlag = kwargs.pop("LFNChecking", True) if checkFlag: argList = list(args) diff --git a/src/DIRAC/Resources/Catalog/test/Test_FileCatalog.py b/src/DIRAC/Resources/Catalog/test/Test_FileCatalog.py index ce618010793..ef385231027 100644 --- a/src/DIRAC/Resources/Catalog/test/Test_FileCatalog.py +++ b/src/DIRAC/Resources/Catalog/test/Test_FileCatalog.py @@ -17,7 +17,6 @@ class GenericCatalog: """Dummy catalog""" def __init__(self, name, nb_read, nb_read_no_lfn, nb_write, nb_write_no_lfn): - self.w_method = [] self.r_method = [] self.no_lfn = [] diff --git a/src/DIRAC/Resources/Cloud/CloudEndpoint.py b/src/DIRAC/Resources/Cloud/CloudEndpoint.py index 5d6686b1aa6..3601904288d 100644 --- a/src/DIRAC/Resources/Cloud/CloudEndpoint.py +++ b/src/DIRAC/Resources/Cloud/CloudEndpoint.py @@ -34,7 +34,6 @@ def __init__(self, parameters=None): self.valid = True def initialize(self): - # Relax security security.SSL_VERSION = ssl.PROTOCOL_SSLv23 security.VERIFY_SSL_CERT = False @@ -292,7 +291,6 @@ def createInstance(self, instanceID="", createPublicIP=True): publicIP = None if createPublicIP: - # Wait until the node is running, otherwise getting public IP fails try: self.__driver.wait_until_running([vmNode], timeout=600) @@ -451,7 +449,6 @@ def stopVM(self, nodeID, publicIP=""): return S_OK() def getVMPool(self, poolName): - try: poolList = self.__driver.ex_list_floating_ip_pools() for pool in poolList: @@ -496,7 +493,6 @@ def assignFloatingIP(self, node): return S_ERROR("No IP pool specified") def getVMFloatingIP(self, publicIP): - # We are still with IPv4 publicIP = publicIP.replace("::ffff:", "") diff --git a/src/DIRAC/Resources/Cloud/EC2Endpoint.py b/src/DIRAC/Resources/Cloud/EC2Endpoint.py index 63f36992b6d..0c9cd9d3fdd 100644 --- a/src/DIRAC/Resources/Cloud/EC2Endpoint.py +++ b/src/DIRAC/Resources/Cloud/EC2Endpoint.py @@ -24,7 +24,6 @@ def __init__(self, parameters=None): self.log.error(result["Message"]) def initialize(self): - availableParams = { "RegionName": "region_name", "AccessKey": "aws_access_key_id", diff --git a/src/DIRAC/Resources/Cloud/Endpoint.py b/src/DIRAC/Resources/Cloud/Endpoint.py index 2d7f9fcc1d2..35a6e85d16d 100644 --- a/src/DIRAC/Resources/Cloud/Endpoint.py +++ b/src/DIRAC/Resources/Cloud/Endpoint.py @@ -45,7 +45,6 @@ def getProxyFileLocation(self): return S_OK(self.proxy) def _createUserDataScript(self): - bootType = self.bootstrapParameters.get("BootType", "pilot") if bootType.lower() == "pilot": return createPilotDataScript(self.parameters, self.bootstrapParameters) diff --git a/src/DIRAC/Resources/Cloud/EndpointFactory.py b/src/DIRAC/Resources/Cloud/EndpointFactory.py index 939d1161fe6..07ddc3155cf 100644 --- a/src/DIRAC/Resources/Cloud/EndpointFactory.py +++ b/src/DIRAC/Resources/Cloud/EndpointFactory.py @@ -11,7 +11,6 @@ def __init__(self): self.log = gLogger.getSubLogger(self.__class__.__name__) def getCE(self, site, endpoint, image=""): - result = getVMTypeConfig(site, endpoint, image) if not result["OK"]: return result diff --git a/src/DIRAC/Resources/Cloud/OcciEndpoint.py b/src/DIRAC/Resources/Cloud/OcciEndpoint.py index 8f3ad114b3d..c55783231db 100644 --- a/src/DIRAC/Resources/Cloud/OcciEndpoint.py +++ b/src/DIRAC/Resources/Cloud/OcciEndpoint.py @@ -70,7 +70,6 @@ def __init__(self, parameters=None): # pprint.pprint( self.scheme ) def initialize(self): - try: result = requests.head(self.serviceUrl + "/-/", headers={"Content-Type": "text/plain"}, **self.authArgs) except Exception as exc: @@ -83,7 +82,6 @@ def initialize(self): return self.__checkConnection() def __getKeystoneUrl(self): - # The URL can be specified in the configuration if self.parameters.get("KeystoneURL"): return S_OK(self.parameters.get("KeystoneURL")) @@ -119,7 +117,6 @@ def __getKeystoneUrl(self): return S_OK(keystoneURL) def __getSchemaDefinitions(self): - try: response = self.session.get("%s/-/" % self.serviceUrl, headers={"Accept": "text/plain,text/occi"}) @@ -234,7 +231,6 @@ def createInstances(self, vmsToSubmit): return S_OK(outputDict) def __renderCategory(self, category, className): - if className not in self.scheme: return None if category not in self.scheme[className]: @@ -454,7 +450,6 @@ def assignFloatingIP(self, nodeID): return S_OK(result.text.split()[1]) def getVMFloatingIP(self, publicIP): - return S_ERROR("Not implemented") def deleteFloatingIP(self, nodeID): diff --git a/src/DIRAC/Resources/Cloud/OpenStackEndpoint.py b/src/DIRAC/Resources/Cloud/OpenStackEndpoint.py index 38b0249961d..c4dc2ae500b 100644 --- a/src/DIRAC/Resources/Cloud/OpenStackEndpoint.py +++ b/src/DIRAC/Resources/Cloud/OpenStackEndpoint.py @@ -75,7 +75,6 @@ def initialize(self): return result def getFlavors(self): - if not self.computeURL or not self.token: return S_ERROR("The endpoint object is not initialized") @@ -95,7 +94,6 @@ def getFlavors(self): return S_OK(self.flavors) def getImages(self): - if not self.imageURL or not self.token: return S_ERROR("The endpoint object is not initialized") @@ -442,7 +440,6 @@ def assignFloatingIP(self, nodeID): return S_OK(ip) def getVMInfo(self, vmID): - try: response = requests.get( f"{self.computeURL}/servers/{vmID}", headers={"X-Auth-Token": self.token}, verify=self.caPath @@ -469,7 +466,6 @@ def getVMInfo(self, vmID): return S_OK(output) def getVMFloatingIP(self, nodeID): - result = self.getVMInfo(nodeID) if not result["OK"]: return result diff --git a/src/DIRAC/Resources/Cloud/RocciEndpoint.py b/src/DIRAC/Resources/Cloud/RocciEndpoint.py index 995da5cc95a..a92bcb1dcbd 100644 --- a/src/DIRAC/Resources/Cloud/RocciEndpoint.py +++ b/src/DIRAC/Resources/Cloud/RocciEndpoint.py @@ -25,7 +25,6 @@ def __init__(self, parameters=None): self.log.error(result["Message"]) def initialize(self): - availableParams = { "EndpointUrl": "endpoint", "Timeout": "timeout", diff --git a/src/DIRAC/Resources/Cloud/Utilities.py b/src/DIRAC/Resources/Cloud/Utilities.py index 57d4fcda138..1e158e5bc79 100644 --- a/src/DIRAC/Resources/Cloud/Utilities.py +++ b/src/DIRAC/Resources/Cloud/Utilities.py @@ -22,7 +22,6 @@ def createMimeData(userDataTuple): - userData = MIMEMultipart() for contents, mtype, fname in userDataTuple: try: @@ -36,7 +35,6 @@ def createMimeData(userDataTuple): def createPilotDataScript(vmParameters, bootstrapParameters): - userDataDict = {} # Arguments to the vm-bootstrap command @@ -183,7 +181,6 @@ def createPilotDataScript(vmParameters, bootstrapParameters): def createUserDataScript(parameters): - defaultUser = os.environ.get("USER", parameters.get("User", "root")) sshUser = parameters.get("SshUser", defaultUser) defaultKey = os.path.expandvars("$HOME/.ssh/id_rsa.pub") diff --git a/src/DIRAC/Resources/Computing/ARC6ComputingElement.py b/src/DIRAC/Resources/Computing/ARC6ComputingElement.py index e6aeefc3c08..da662e3e5b6 100755 --- a/src/DIRAC/Resources/Computing/ARC6ComputingElement.py +++ b/src/DIRAC/Resources/Computing/ARC6ComputingElement.py @@ -25,7 +25,7 @@ import arc # Has to work if this module is called #pylint: disable=import-error from DIRAC import S_OK, S_ERROR -from DIRAC.Resources.Computing.ARCComputingElement import ARCComputingElement +from DIRAC.Resources.Computing.ARCComputingElement import ARCComputingElement, prepareProxyToken class ARC6ComputingElement(ARCComputingElement): @@ -89,17 +89,13 @@ def _getARCJob(self, jobID): j.PrepareHandler(self.usercfg) return j + @prepareProxyToken def submitJob(self, executableFile, proxy, numberOfJobs=1): """Method to submit job""" # Assume that the ARC queues are always of the format nordugrid-- # And none of our supported batch systems have a "-" in their name self.arcQueue = self.queue.split("-", 2)[2] - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): @@ -136,7 +132,6 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1): break for __i in range(numberOfJobs): - # The basic job description jobdescs = arc.JobDescriptionList() @@ -178,17 +173,12 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1): result = S_ERROR("No pilot references obtained from the ARC job submission") return result + @prepareProxyToken def getCEStatus(self): """Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) - # Creating an endpoint endpoint = arc.Endpoint(self.ceHost, arc.Endpoint.COMPUTINGINFO, self.computingInfoEndpoint) diff --git a/src/DIRAC/Resources/Computing/ARCComputingElement.py b/src/DIRAC/Resources/Computing/ARCComputingElement.py index b293fea822a..cf2805116ab 100755 --- a/src/DIRAC/Resources/Computing/ARCComputingElement.py +++ b/src/DIRAC/Resources/Computing/ARCComputingElement.py @@ -51,7 +51,6 @@ import arc # Has to work if this module is called #pylint: disable=import-error from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping from DIRAC.Core.Utilities.Subprocess import shellCall from DIRAC.Core.Utilities.File import makeGuid from DIRAC.Core.Utilities.List import breakListIntoChunks @@ -88,8 +87,30 @@ } -class ARCComputingElement(ComputingElement): +def prepareProxyToken(func): + """Decorator to set up proxy or token as necessary""" + + def wrapper(*args, **kwargs): + # Get the reference to the CE class object + self = args[0] + + # Prepare first the proxy + result = self._prepareProxy() + if not result["OK"]: + self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) + return result + self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) + + # Set the token in the environment if needed + if self.token: + os.environ["BEARER_TOKEN"] = self.token["access_token"] + + return func(*args, **kwargs) + + return wrapper + +class ARCComputingElement(ComputingElement): _arcLevels = ["DEBUG", "VERBOSE", "INFO", "WARNING", "ERROR", "FATAL"] ############################################################################# @@ -284,13 +305,9 @@ def _reset(self): return S_OK() ############################################################################# + @prepareProxyToken def submitJob(self, executableFile, proxy, numberOfJobs=1, inputs=None, outputs=None): """Method to submit job""" - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): @@ -378,15 +395,10 @@ def _analyzeSubmissionError(self, result): self.log.warn("%s ... maybe above messages will give a hint." % message) ############################################################################# + @prepareProxyToken def killJob(self, jobIDList): """Kill the specified jobs""" - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) - jobList = list(jobIDList) if isinstance(jobIDList, str): jobList = [jobIDList] @@ -407,17 +419,12 @@ def killJob(self, jobIDList): return S_OK() ############################################################################# + @prepareProxyToken def getCEStatus(self): """Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) - # Try to find out which VO we are running for. vo = "" res = getVOfromProxyGroup() @@ -426,6 +433,7 @@ def getCEStatus(self): result = S_OK() result["SubmittedJobs"] = 0 + if not vo: # Presumably the really proper way forward once the infosys-discuss WG comes up with a solution # and it is implemented. Needed for DIRAC instances which use robot certificates for pilots. @@ -486,15 +494,10 @@ def getCEStatus(self): return result ############################################################################# + @prepareProxyToken def getJobStatus(self, jobIDList): """Get the status information for the given list of jobs""" - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) - jobTmpList = list(jobIDList) if isinstance(jobIDList, str): jobTmpList = [jobIDList] @@ -571,16 +574,12 @@ def getJobStatus(self, jobIDList): return S_OK(resultDict) ############################################################################# + @prepareProxyToken def getJobOutput(self, jobID, workingDirectory=None): """Get the specified job standard output and error files. Standard output and error are returned as strings. If further outputs are retrieved, they are stored in workingDirectory. """ - result = self._prepareProxy() - if not result["OK"]: - self.log.error("ARCComputingElement: failed to set up proxy", result["Message"]) - return result - self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"]) if jobID.find(":::") != -1: pilotRef, stamp = jobID.split(":::") diff --git a/src/DIRAC/Resources/Computing/AREXComputingElement.py b/src/DIRAC/Resources/Computing/AREXComputingElement.py index 7f1dec6cf1e..df132eec2f5 100755 --- a/src/DIRAC/Resources/Computing/AREXComputingElement.py +++ b/src/DIRAC/Resources/Computing/AREXComputingElement.py @@ -93,6 +93,17 @@ def _reset(self): ############################################################################# + def setToken(self, token, valid): + """Set the token and update the headers + + :param token: OAuth2Token object or dictionary containing token structure + :param int valid: validity period in seconds + :return: S_OK + """ + super().setToken(token, valid) + self.headers["Authorization"] = "Bearer " + self.token["access_token"] + return S_OK() + def _arcToDiracID(self, arcJobID): """Convert an ARC jobID into a DIRAC jobID. Example: 1234 becomes https://:/arex/1234 @@ -731,7 +742,7 @@ def getJobOutput(self, jobID, workingDirectory=None): if ":::" in jobID: jobRef, stamp = jobID.split(":::") else: - return S_ERROR(f"DIRAC stamp not defined for {jobRef}") + return S_ERROR(f"DIRAC stamp not defined for {jobID}") job = self._DiracToArcID(jobRef) # Get the list of available outputs diff --git a/src/DIRAC/Resources/Computing/BOINCComputingElement.py b/src/DIRAC/Resources/Computing/BOINCComputingElement.py index f0f2754a4ae..14d98e1025e 100644 --- a/src/DIRAC/Resources/Computing/BOINCComputingElement.py +++ b/src/DIRAC/Resources/Computing/BOINCComputingElement.py @@ -15,7 +15,6 @@ class BOINCComputingElement(ComputingElement): - ############################################################################### def __init__(self, ceUniqueID): """Standard constructor.""" @@ -311,7 +310,6 @@ def _fromStrToFile(self, strContent, fileName): # testing this if __name__ == "__main__": - test_boinc = BOINCComputingElement(12) test_submit = 1 test_getStatus = 2 diff --git a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/SGEResourceUsage.py b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/SGEResourceUsage.py index 5baa6fd21a7..2df7885c228 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/SGEResourceUsage.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/SGEResourceUsage.py @@ -111,7 +111,6 @@ def getResourceUsage(self): def _getCPUScalingFactor(): - host = socket.getfqdn() cmd = "qconf -se %s" % host result = runCommand(cmd) diff --git a/src/DIRAC/Resources/Computing/CREAMComputingElement.py b/src/DIRAC/Resources/Computing/CREAMComputingElement.py index 32ae786370e..c13730e86a9 100644 --- a/src/DIRAC/Resources/Computing/CREAMComputingElement.py +++ b/src/DIRAC/Resources/Computing/CREAMComputingElement.py @@ -30,7 +30,6 @@ class CREAMComputingElement(ComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" diff --git a/src/DIRAC/Resources/Computing/ComputingElement.py b/src/DIRAC/Resources/Computing/ComputingElement.py index d25de615d6a..55416568335 100755 --- a/src/DIRAC/Resources/Computing/ComputingElement.py +++ b/src/DIRAC/Resources/Computing/ComputingElement.py @@ -61,6 +61,7 @@ def __init__(self, ceName): self.ceName = ceName self.ceParameters = {} self.proxy = "" + self.token = None self.valid = None self.mandatoryParameters = [] self.batchSystem = None @@ -85,6 +86,11 @@ def setProxy(self, proxy, valid=0): self.valid = datetime.datetime.utcnow() + second * valid return S_OK() + def setToken(self, token, valid=0): + self.token = token + self.valid = datetime.datetime.utcnow() + second * valid + return S_OK() + def _prepareProxy(self): """Set the environment variable X509_USER_PROXY""" if not self.proxy: diff --git a/src/DIRAC/Resources/Computing/ComputingElementFactory.py b/src/DIRAC/Resources/Computing/ComputingElementFactory.py index 405d3a9481e..7b4d8f55c17 100755 --- a/src/DIRAC/Resources/Computing/ComputingElementFactory.py +++ b/src/DIRAC/Resources/Computing/ComputingElementFactory.py @@ -7,7 +7,6 @@ class ComputingElementFactory: - ############################################################################# def __init__(self, ceType=""): """Standard constructor""" diff --git a/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py b/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py index e141f9f3e19..7e87e7b442e 100644 --- a/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py +++ b/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py @@ -36,7 +36,7 @@ When not using a local condor_schedd, add ``delegate_job_GSI_credentials_lifetime = 0`` to the ``ExtraSubmitString``. -When using a local condor_schedd look at the HTCondor documenation for enabling the proxy refresh. +When using a local condor_schedd look at the HTCondor documentation for enabling the proxy refresh. **Code Documentation** """ @@ -51,6 +51,7 @@ import datetime import errno import threading +import textwrap from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.Resources.Computing.ComputingElement import ComputingElement @@ -60,8 +61,9 @@ from DIRAC.WorkloadManagementSystem.Client import PilotStatus from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient from DIRAC.Core.Utilities.File import makeGuid +from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import writeToTokenFile -from DIRAC.Resources.Computing.BatchSystems.Condor import parseCondorStatus, treatCondorHistory +from DIRAC.Resources.Computing.BatchSystems.Condor import parseCondorStatus MANDATORY_PARAMETERS = ["Queue"] DEFAULT_WORKINGDIRECTORY = "/opt/dirac/pro/runit/WorkloadManagement/SiteDirectorHT" @@ -145,9 +147,10 @@ def __init__(self, ceUniqueID): ) self.useLocalSchedd = True self.remoteScheddOptions = "" + self.tokenFile = None ############################################################################# - def __writeSub(self, executable, nJobs, location, processors): + def __writeSub(self, executable, nJobs, location, processors, tokenFile=None): """Create the Sub File for submission. :param str executable: name of the script to execute @@ -168,6 +171,16 @@ def __writeSub(self, executable, nJobs, location, processors): executable = os.path.join(self.workingDirectory, executable) + useCredentials = "" + if tokenFile: + useCredentials = textwrap.dedent( + """ + use_scitokens = true + scitokens_file = %s + """ + % tokenFile + ) + # This is used to remove outputs from the remote schedd # Used in case a local schedd is not used periodicRemove = "periodic_remove = " @@ -189,6 +202,7 @@ def __writeSub(self, executable, nJobs, location, processors): executable = %(executable)s universe = %(targetUniverse)s use_x509userproxy = true +%(useCredentials)s output = $(Cluster).$(Process).out error = $(Cluster).$(Process).err log = $(Cluster).$(Process).log @@ -214,6 +228,7 @@ def __writeSub(self, executable, nJobs, location, processors): initialDir=os.path.join(self.workingDirectory, location), localScheddOptions=localScheddOptions, targetUniverse=targetUniverse, + useCredentials=useCredentials, ) subFile.write(sub) subFile.close() @@ -237,6 +252,38 @@ def _reset(self): self.log.debug("Remote scheduler option:", self.remoteScheddOptions) return S_OK() + def _executeCondorCommand(self, cmd, keepTokenFile=False): + tFile = None + if self.token: + # Create a new token file if we do not keep it across several calls + if keepTokenFile: + tFile = self.tokenFile + if not tFile: + tFile = tempfile.NamedTemporaryFile(suffix=".token", prefix="HTCondorCE_", dir=self.workingDirectory) + writeToTokenFile(self.token["access_token"], tFile.name) + + htcEnv = { + "_CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS": "SCITOKENS", + "_CONDOR_SCITOKENS_FILE": tFile.name, + } + else: + htcEnv = {"_CONDOR_SEC_CLIENT_AUTHENTICATION_METHODS": "GSI"} + + result = executeGridCommand( + self.proxy, + cmd, + gridEnvScript=self.gridEnv, + gridEnvDict=htcEnv, + ) + # Remove token file if we do not want to keep it + if tFile: + if keepTokenFile: + self.tokenFile = tFile + else: + self.tokenFile = None + + return result + ############################################################################# def submitJob(self, executableFile, proxy, numberOfJobs=1): """Method to submit job""" @@ -256,6 +303,7 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1): # We randomize the location of the pilot output and log, because there are just too many of them location = logDir(self.ceName, commonJobStampPart) nProcessors = self.ceParameters.get("NumberOfProcessors", 1) + subName = self.__writeSub(executableFile, numberOfJobs, location, nProcessors) cmd = ["condor_submit", "-terse", subName] @@ -265,19 +313,21 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1): for op in scheddOptions: cmd.insert(-1, op) - result = executeGridCommand(self.proxy, cmd, self.gridEnv) + result = self._executeCondorCommand(cmd) self.log.verbose(result) os.remove(subName) if not result["OK"]: self.log.error("Failed to submit jobs to htcondor", result["Message"]) return result - if result["Value"][0]: + status, stdout, stderr = result["Value"] + + if status: # We have got a non-zero status code - errorString = result["Value"][2] if result["Value"][2] else result["Value"][1] + errorString = stderr if stderr else stdout return S_ERROR(f"Pilot submission failed with error: {errorString.strip()}") - pilotJobReferences = self.__getPilotReferences(result["Value"][1].strip()) + pilotJobReferences = self.__getPilotReferences(stdout.strip()) if not pilotJobReferences["OK"]: return pilotJobReferences pilotJobReferences = pilotJobReferences["Value"] @@ -304,20 +354,26 @@ def killJob(self, jobIDList): self.log.verbose("KillJob jobIDList", jobIDList) + self.tokenFile = None + for jobRef in jobIDList: job, _, jobID = condorIDAndPathToResultFromJobRef(jobRef) self.log.verbose("Killing pilot", job) cmd = ["condor_rm"] cmd.extend(self.remoteScheddOptions.strip().split(" ")) cmd.append(jobID) - result = executeGridCommand(self.proxy, cmd, self.gridEnv) + result = self._executeCondorCommand(cmd, keepTokenFile=True) if not result["OK"]: + self.tokenFile = None return S_ERROR(f"condor_rm failed completely: {result['Message']}") status, stdout, stderr = result["Value"] if status != 0: self.log.warn("Failed to kill pilot", f"{job}: {stdout}, {stderr}") + self.tokenFile = None return S_ERROR(f"Failed to kill pilot {job}: {stderr}") + self.tokenFile = None + return S_OK() ############################################################################# @@ -367,42 +423,73 @@ def getJobStatus(self, jobIDList): job, _, jobID = condorIDAndPathToResultFromJobRef(jobRef) condorIDs[job] = jobID + self.tokenFile = None + qList = [] for _condorIDs in breakListIntoChunks(condorIDs.values(), 100): - # This will return a list of 1245.75 3 - status, stdout_q = subprocess.getstatusoutput( - "condor_q {} {} -af:j JobStatus ".format(self.remoteScheddOptions, " ".join(_condorIDs)) - ) + cmd = ["condor_q"] + cmd.extend(self.remoteScheddOptions.strip().split(" ")) + cmd.extend(_condorIDs) + cmd.extend(["-af:j", "JobStatus"]) + result = self._executeCondorCommand(cmd, keepTokenFile=True) + if not result["OK"]: + self.tokenFile = None + return S_ERROR("condor_q failed completely: %s" % result["Message"]) + status, stdout, stderr = result["Value"] if status != 0: - return S_ERROR(stdout_q) - _qList = stdout_q.strip().split("\n") + self.tokenFile = None + return S_ERROR(stdout + stderr) + _qList = stdout.strip().split("\n") qList.extend(_qList) # FIXME: condor_history does only support j for autoformat from 8.5.3, # format adds whitespace for each field This will return a list of 1245 75 3 - # needs to cocatenate the first two with a dot + # needs to concatenate the first two with a dot condorHistCall = "condor_history {} {} -af ClusterId ProcId JobStatus".format( self.remoteScheddOptions, " ".join(_condorIDs), - ) + ).split() - treatCondorHistory(condorHistCall, qList) + self._treatCondorHistory(condorHistCall, qList) for job, jobID in condorIDs.items(): - pilotStatus = parseCondorStatus(qList, jobID) if pilotStatus == "HELD": # make sure the pilot stays dead and gets taken out of the condor_q - _rmStat, _rmOut = subprocess.getstatusoutput(f"condor_rm {self.remoteScheddOptions} {jobID} ") - # self.log.debug( "condor job killed: job %s, stat %s, message %s " % ( jobID, rmStat, rmOut ) ) + cmd = f"condor_rm {self.remoteScheddOptions} {jobID}".split() + _result = self._executeCondorCommand(cmd, keepTokenFile=True) pilotStatus = PilotStatus.ABORTED resultDict[job] = pilotStatus + self.tokenFile = None + self.log.verbose("Pilot Statuses: %s " % resultDict) return S_OK(resultDict) + def _treatCondorHistory(self, condorHistCall, qList): + """concatenate clusterID and processID to get the same output as condor_q + until we can expect condor version 8.5.3 everywhere + + :param str condorHistCall: condor_history command to run + :param list qList: list of jobID and status from condor_q output, will be modified in this function + :returns: None + """ + + result = self._executeCondorCommand(condorHistCall) + if not result["OK"]: + return S_ERROR("condorHistCall failed completely: %s" % result["Message"]) + + status_history, stdout_history, stderr_history = result["Value"] + + # Join the ClusterId and the ProcId and add to existing list of statuses + if status_history == 0: + for line in stdout_history.split("\n"): + values = line.strip().split() + if len(values) == 3: + qList.append("%s.%s %s" % tuple(values)) + def getJobLog(self, jobID): """Get pilot job logging info from HTCondor @@ -447,7 +534,7 @@ def __getJobOutput(self, jobID, outTypes): if not self.useLocalSchedd: cmd = ["condor_transfer_data", "-pool", f"{self.ceName}:9619", "-name", self.ceName, condorID] - result = executeGridCommand(self.proxy, cmd, self.gridEnv) + result = self._executeCondorCommand(cmd) self.log.verbose(result) # Getting 'logging' without 'error' and 'output' is possible but will generate command errors @@ -458,9 +545,10 @@ def __getJobOutput(self, jobID, outTypes): self.log.error(errorMessage, result["Message"]) return result # Even if result is OK, the actual exit code of cmd can still be an error - if result["OK"] and result["Value"][0] != 0: - outMessage = result["Value"][1].strip() - errMessage = result["Value"][2].strip() + status, stdout, stderr = result["Value"] + if status != 0: + outMessage = stdout.strip() + errMessage = stderr.strip() varMessage = outMessage + " " + errMessage self.log.error(errorMessage, varMessage) return S_ERROR(f"{errorMessage}: {varMessage}") diff --git a/src/DIRAC/Resources/Computing/InProcessComputingElement.py b/src/DIRAC/Resources/Computing/InProcessComputingElement.py index 99a7f338c8c..599e08ce63c 100755 --- a/src/DIRAC/Resources/Computing/InProcessComputingElement.py +++ b/src/DIRAC/Resources/Computing/InProcessComputingElement.py @@ -14,7 +14,6 @@ class InProcessComputingElement(ComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" diff --git a/src/DIRAC/Resources/Computing/PoolComputingElement.py b/src/DIRAC/Resources/Computing/PoolComputingElement.py index c36ee128656..ed958add7aa 100644 --- a/src/DIRAC/Resources/Computing/PoolComputingElement.py +++ b/src/DIRAC/Resources/Computing/PoolComputingElement.py @@ -61,7 +61,6 @@ def executeJob(executableFile, proxy, taskID, inputs, **kwargs): class PoolComputingElement(ComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" diff --git a/src/DIRAC/Resources/Computing/SSHBatchComputingElement.py b/src/DIRAC/Resources/Computing/SSHBatchComputingElement.py index 70d6558c03b..1f53f951a16 100644 --- a/src/DIRAC/Resources/Computing/SSHBatchComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHBatchComputingElement.py @@ -13,7 +13,6 @@ class SSHBatchComputingElement(SSHComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" @@ -124,7 +123,6 @@ def killJob(self, jobIDs): hostDict = {} for job in jobIDList: - host = os.path.dirname(urlparse(job).path).lstrip("/") hostDict.setdefault(host, []) hostDict[host].append(job) diff --git a/src/DIRAC/Resources/Computing/SSHComputingElement.py b/src/DIRAC/Resources/Computing/SSHComputingElement.py index 952ec9e5f6b..63d8e7ee48e 100644 --- a/src/DIRAC/Resources/Computing/SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHComputingElement.py @@ -110,7 +110,6 @@ class SSH: """ def __init__(self, host=None, parameters=None): - self.host = host if parameters is None: parameters = {} @@ -134,7 +133,6 @@ def __init__(self, host=None, parameters=None): self.log = gLogger.getSubLogger("SSH") def __ssh_call(self, command, timeout): - if not timeout: timeout = 999 @@ -306,7 +304,6 @@ def scpCall(self, timeout, localFile, remoteFile, postUploadCommand="", upload=T class SSHComputingElement(ComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" @@ -490,7 +487,6 @@ def _generateControlScript(self): return S_OK("%s" % controlScript) def __executeHostCommand(self, command, options, ssh=None, host=None): - if not ssh: ssh = SSH(host=host, parameters=self.ceParameters) @@ -545,7 +541,6 @@ def __executeHostCommand(self, command, options, ssh=None, host=None): return S_ERROR("\n".join([sshStdout, sshStderr])) def submitJob(self, executableFile, proxy, numberOfJobs=1): - # self.log.verbose( "Executable file path: %s" % executableFile ) if not os.access(executableFile, 5): os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) diff --git a/src/DIRAC/Resources/Computing/SudoComputingElement.py b/src/DIRAC/Resources/Computing/SudoComputingElement.py index 8c540174f3c..20de10a8286 100644 --- a/src/DIRAC/Resources/Computing/SudoComputingElement.py +++ b/src/DIRAC/Resources/Computing/SudoComputingElement.py @@ -14,7 +14,6 @@ class SudoComputingElement(ComputingElement): - ############################################################################# def __init__(self, ceUniqueID): """Standard constructor.""" diff --git a/src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py b/src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py index 4b5706efaa4..a0b50bc5d72 100644 --- a/src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py +++ b/src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py @@ -59,10 +59,14 @@ def test_parseCondorStatus(): def test_getJobStatus(mocker): """Test HTCondorCE getJobStatus""" - mocker.patch(MODNAME + ".subprocess.getstatusoutput", side_effect=([(0, "\n".join(STATUS_LINES)), (0, 0)])) - patchPopen = mocker.patch("DIRAC.Resources.Computing.BatchSystems.Condor.subprocess.Popen") - patchPopen.return_value.communicate.side_effect = [("\n".join(HISTORY_LINES), "")] - patchPopen.return_value.returncode = 0 + mocker.patch( + MODNAME + ".executeGridCommand", + side_effect=[ + S_OK((0, "\n".join(STATUS_LINES), "")), + S_OK((0, "\n".join(HISTORY_LINES), "")), + S_OK((0, "", "")), + ], + ) mocker.patch(MODNAME + ".HTCondorCEComputingElement._HTCondorCEComputingElement__cleanup") htce = HTCE.HTCondorCEComputingElement(12345) @@ -159,7 +163,7 @@ def test_submitJob(setUp, mocker, localSchedd, expected): ceName = "condorce.cern.ch" htce.ceName = ceName - execMock = mocker.patch(MODNAME + ".executeGridCommand", return_value=S_OK((0, "123.0 - 123.0"))) + execMock = mocker.patch(MODNAME + ".executeGridCommand", return_value=S_OK((0, "123.0 - 123.0", ""))) mocker.patch( MODNAME + ".HTCondorCEComputingElement._HTCondorCEComputingElement__writeSub", return_value="dirac_pilot" ) diff --git a/src/DIRAC/Resources/Computing/test/Test_PoolComputingElement.py b/src/DIRAC/Resources/Computing/test/Test_PoolComputingElement.py index 2de350f510e..1e6cb02d36b 100644 --- a/src/DIRAC/Resources/Computing/test/Test_PoolComputingElement.py +++ b/src/DIRAC/Resources/Computing/test/Test_PoolComputingElement.py @@ -66,7 +66,6 @@ def createAndDelete(): @pytest.mark.slow def test_submit_and_shutdown(createAndDelete): - time.sleep(0.5) ceParameters = {"WholeNode": True, "NumberOfProcessors": 4} @@ -83,7 +82,6 @@ def test_submit_and_shutdown(createAndDelete): def test_executeJob_wholeNode4(createAndDelete): - time.sleep(0.5) ceParameters = {"WholeNode": True, "NumberOfProcessors": 4} @@ -114,7 +112,6 @@ def test_executeJob_wholeNode4(createAndDelete): @pytest.mark.slow def test_executeJob_wholeNode8(createAndDelete): - time.sleep(0.5) ceParameters = {"WholeNode": True, "NumberOfProcessors": 8} @@ -159,7 +156,6 @@ def test_executeJob_wholeNode8(createAndDelete): def test_executeJob_submitAndStop(createAndDelete): - time.sleep(0.5) ceParameters = {"WholeNode": True, "NumberOfProcessors": 4} @@ -187,7 +183,6 @@ def test_executeJob_submitAndStop(createAndDelete): @pytest.mark.slow def test_executeJob_WholeNodeJobs(createAndDelete): - time.sleep(0.5) ce = PoolComputingElement("TestPoolCE") diff --git a/src/DIRAC/Resources/IdProvider/CheckInIdProvider.py b/src/DIRAC/Resources/IdProvider/CheckInIdProvider.py index a8d4f3cda81..20dad7f030d 100644 --- a/src/DIRAC/Resources/IdProvider/CheckInIdProvider.py +++ b/src/DIRAC/Resources/IdProvider/CheckInIdProvider.py @@ -2,7 +2,6 @@ """ from authlib.oauth2.rfc6749.util import scope_to_list -from DIRAC import S_OK from DIRAC.Resources.IdProvider.OAuth2IdProvider import OAuth2IdProvider from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup, getGroupOption @@ -21,4 +20,4 @@ def getGroupScopes(self, group): getVOForGroup(group), group.split("_")[1], ) - return S_OK(scope_to_list(idPScope)) + return scope_to_list(idPScope) diff --git a/src/DIRAC/Resources/IdProvider/IAMIdProvider.py b/src/DIRAC/Resources/IdProvider/IAMIdProvider.py index 40ecfc40dbf..d3828c5105c 100644 --- a/src/DIRAC/Resources/IdProvider/IAMIdProvider.py +++ b/src/DIRAC/Resources/IdProvider/IAMIdProvider.py @@ -2,7 +2,6 @@ """ from authlib.oauth2.rfc6749.util import scope_to_list -from DIRAC import S_OK from DIRAC.Resources.IdProvider.OAuth2IdProvider import OAuth2IdProvider from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup, getGroupOption @@ -18,4 +17,4 @@ def getGroupScopes(self, group): idPScope = getGroupOption(group, "IdPRole") if not idPScope: idPScope = "wlcg.groups:/{}/{}".format(getVOForGroup(group), group.split("_")[1]) - return S_OK(scope_to_list(idPScope)) + return scope_to_list(idPScope) diff --git a/src/DIRAC/Resources/IdProvider/IdProvider.py b/src/DIRAC/Resources/IdProvider/IdProvider.py deleted file mode 100644 index 99dd5d72ff2..00000000000 --- a/src/DIRAC/Resources/IdProvider/IdProvider.py +++ /dev/null @@ -1,28 +0,0 @@ -""" IdProvider base class for various identity providers -""" -from DIRAC import gLogger - - -class IdProvider: - - DEFAULT_METADATA = {} - - def __init__(self, **kwargs): - """C'or""" - self.log = gLogger.getSubLogger(self.__class__.__name__) - meta = self.DEFAULT_METADATA - meta.update(kwargs) - self.setParameters(meta) - self._initialization(**meta) - - def _initialization(self, **kwargs): - """Initialization""" - pass - - def setParameters(self, parameters): - """Set parameters - - :param dict parameters: parameters of the identity Provider - """ - self.parameters = parameters - self.name = parameters.get("ProviderName") diff --git a/src/DIRAC/Resources/IdProvider/IdProviderFactory.py b/src/DIRAC/Resources/IdProvider/IdProviderFactory.py index a7302c16cf5..96197a299f2 100644 --- a/src/DIRAC/Resources/IdProvider/IdProviderFactory.py +++ b/src/DIRAC/Resources/IdProvider/IdProviderFactory.py @@ -25,13 +25,13 @@ def __init__(self): self.cacheMetadata = DictCache() @gCacheMetadata - def getMetadata(self, idP): - return self.cacheMetadata.get(idP) or {} + def getMetadata(self, idProvider): + return self.cacheMetadata.get(idProvider) or {} @gCacheMetadata - def addMetadata(self, idP, data, time=24 * 3600): + def addMetadata(self, idProvider, data, time=24 * 3600): if data: - self.cacheMetadata.add(idP, time, data) + self.cacheMetadata.add(idProvider, time, data) def getIdProviderForToken(self, token): """This method returns a IdProvider instance corresponding to the supplied @@ -72,7 +72,7 @@ def getIdProvider(self, name, **kwargs): self.log.debug("Search configuration for", name) clients = getDIRACClients() if name in clients: - # If it is a DIRAC default pre-registred client + # If it is a DIRAC default pre-registered client pDict = asMetaDict pDict.update(clients[name]) else: diff --git a/src/DIRAC/Resources/IdProvider/OAuth2IdProvider.py b/src/DIRAC/Resources/IdProvider/OAuth2IdProvider.py index af265ece961..330720bc155 100644 --- a/src/DIRAC/Resources/IdProvider/OAuth2IdProvider.py +++ b/src/DIRAC/Resources/IdProvider/OAuth2IdProvider.py @@ -14,16 +14,14 @@ from authlib.oidc.discovery.well_known import get_well_known_url from authlib.oauth2.rfc7636 import create_s256_code_challenge - -from DIRAC import S_OK, S_ERROR +from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Utilities import ThreadSafe -from DIRAC.Resources.IdProvider.IdProvider import IdProvider from DIRAC.ConfigurationSystem.Client.Helpers.Registry import ( getVOMSRoleGroupMapping, - getGroupOption, - getAllGroups, wrapIDAsDN, getVOs, + getGroupOption, + getAllGroups, ) from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import OAuth2Token from DIRAC.FrameworkSystem.private.authorization.utils.Requests import createOAuth2Request @@ -44,7 +42,6 @@ def claimParser(claimDict, attributes): :return: dict """ profile = {} - result = None for claim, reg in attributes.items(): if claim not in claimDict: continue @@ -149,16 +146,17 @@ def _exchange_token(self, url, body="", refresh_token=None, headers=None, auth=N return self.token -class OAuth2IdProvider(IdProvider, OAuth2Session): +class OAuth2IdProvider(OAuth2Session): """Base class to describe the configuration of the OAuth2 client of the corresponding provider.""" JWKS_REFRESH_RATE = 24 * 3600 METADATA_REFRESH_RATE = 24 * 3600 + DEFAULT_METADATA = {} def __init__(self, **kwargs): """Initialization""" - IdProvider.__init__(self, **kwargs) OAuth2Session.__init__(self, **kwargs) + self.log = gLogger.getSubLogger(self.__class__.__name__) self.metadata_fetch_last = 0 self.issuer = self.metadata["issuer"] self.scope = self.scope or "" @@ -169,6 +167,10 @@ def __init__(self, **kwargs): # self.token_endpoint_auth_method = kwargs.get('token_endpoint_auth_method') #, 'client_secret_post') self.server_metadata_url = kwargs.get("server_metadata_url", get_well_known_url(self.metadata["issuer"], True)) self.jwks_fetch_last = time.time() - self.JWKS_REFRESH_RATE + meta = self.DEFAULT_METADATA + meta.update(kwargs) + self.setParameters(meta) + self._initialize() self.metadata_fetch_last = time.time() - self.METADATA_REFRESH_RATE self.log.debug( '"%s" OAuth2 IdP initialization done:' % self.name, @@ -176,6 +178,17 @@ def __init__(self, **kwargs): % (self.client_id, self.client_secret, pprint.pformat(self.metadata)), ) + def _initialize(self): + """Initialization""" + self.name = self.parameters.get("ProviderName") + + def setParameters(self, parameters: dict): + """Set parameters + + :param dict parameters: parameters of the identity Provider + """ + self.parameters = parameters + def get_metadata(self, option=None): """Get metadata @@ -189,12 +202,28 @@ def get_metadata(self, option=None): @gMetadata def fetch_metadata(self): - """Fetch metada""" + """Fetch metadata""" if self.metadata_fetch_last < (time.time() - self.METADATA_REFRESH_RATE): data = self.get(self.server_metadata_url, withhold_token=True).json() self.metadata.update(data) self.metadata_fetch_last = time.time() + def getGroupScopes(self, group: str) -> list: + """Get group scopes + + :param group: DIRAC group + """ + idPScope = getGroupOption(group, "IdPRole") + return scope_to_list(idPScope) if idPScope else [] + + def getScopeGroups(self, scope: str) -> list: + """Get DIRAC groups related to scope""" + groups = [] + for group in getAllGroups(): + if (g_scope := self.getGroupScopes(group)) and set(g_scope).issubset(scope_to_list(scope)): + groups.append(group) + return groups + @gJWKs def updateJWKs(self): """Update JWKs""" @@ -338,7 +367,7 @@ def researchGroup(self, payload=None, token=None): if not credDict.get("DIRACGroups"): credDict.update(self.parseEduperson(payload)) if credDict.get("DIRACGroups"): - self.log.debug("Found next groups:", ", ".join(credDict["DIRACGroups"])) + self.log.debug("Found groups:", ", ".join(credDict["DIRACGroups"])) credDict["group"] = credDict["DIRACGroups"][0] return S_OK(credDict) @@ -399,7 +428,7 @@ def parseEduperson(self, claimDict): return credDict def deviceAuthorization(self, group=None): - """Authorizaion through DeviceCode flow""" + """Authorization through DeviceCode flow""" result = self.submitDeviceCodeAuthorizationFlow(group) if not result["OK"]: return result @@ -527,29 +556,13 @@ def waitFinalStatusOfDeviceCodeAuthorizationFlow(self, deviceCode, interval=5, t ) token = r.json() if not token: - return S_ERROR("Resived token is empty!") + return S_ERROR("Received token is empty!") if "error" not in token: self.token = token return S_OK(token) if token["error"] != "authorization_pending": return S_ERROR((token.get("error") or "unknown") + " : " + (token.get("error_description") or "")) - def getGroupScopes(self, group: str) -> list: - """Get group scopes - - :param group: DIRAC group - """ - idPScope = getGroupOption(group, "IdPRole") - return scope_to_list(idPScope) if idPScope else [] - - def getScopeGroups(self, scope: str) -> list: - """Get DIRAC groups related to scope""" - groups = [] - for group in getAllGroups(): - if (g_scope := self.getGroupScopes(group)) and set(g_scope).issubset(scope_to_list(scope)): - groups.append(group) - return groups - def getUserProfile(self): """Get user profile diff --git a/src/DIRAC/Resources/MessageQueue/MQConnectionManager.py b/src/DIRAC/Resources/MessageQueue/MQConnectionManager.py index 3279c262a66..d392edd803f 100644 --- a/src/DIRAC/Resources/MessageQueue/MQConnectionManager.py +++ b/src/DIRAC/Resources/MessageQueue/MQConnectionManager.py @@ -81,6 +81,7 @@ def addNewMessenger(self, mqURI, messengerType): S_OK: with the value of the messenger Id or S_ERROR if the messenger was not added, cause the same id already exists. """ + # 'consumer1' ->1 # 'producer21' ->21 def msgIdToInt(msgIds, msgType): diff --git a/src/DIRAC/Resources/MessageQueue/Simple/StompInterface.py b/src/DIRAC/Resources/MessageQueue/Simple/StompInterface.py index fa55271391d..526df164f1f 100644 --- a/src/DIRAC/Resources/MessageQueue/Simple/StompInterface.py +++ b/src/DIRAC/Resources/MessageQueue/Simple/StompInterface.py @@ -396,7 +396,6 @@ def createProducer( csDestinations = brokerParams.pop("destinations", []) if not destination: - if len(csDestinations) != 1: raise ValueError("There should be exactly one destination given in parameter or in the CS") destination = csDestinations[0] diff --git a/src/DIRAC/Resources/MessageQueue/StompMQConnector.py b/src/DIRAC/Resources/MessageQueue/StompMQConnector.py index e06695f27c3..2c9d164d0a0 100644 --- a/src/DIRAC/Resources/MessageQueue/StompMQConnector.py +++ b/src/DIRAC/Resources/MessageQueue/StompMQConnector.py @@ -113,7 +113,6 @@ def setupConnection(self, parameters=None): return S_ERROR(EMQCONN, "Invalid SSL version provided: %s" % sslVersion) try: - # Get IP addresses of brokers # Start with the IPv6, and randomize it ipv6_addrInfo = socket.getaddrinfo(host, port, socket.AF_INET6, socket.SOCK_STREAM) diff --git a/src/DIRAC/Resources/MessageQueue/test/Test_MQ_Utilities.py b/src/DIRAC/Resources/MessageQueue/test/Test_MQ_Utilities.py index 358fbad09d7..e74a1bc0e0c 100644 --- a/src/DIRAC/Resources/MessageQueue/test/Test_MQ_Utilities.py +++ b/src/DIRAC/Resources/MessageQueue/test/Test_MQ_Utilities.py @@ -94,7 +94,6 @@ class Test_getMQParamFromCSSuccessTestCase(unittest.TestCase): """Test class to check success scenarios.""" def setUp(self): - # external dependencies module.CSAPI = MagicMock() module.gConfig = MagicMock() @@ -105,7 +104,6 @@ def setUp(self): ] def test_getQueue(self): - module.gConfig.getConfigurationTree.return_value = {"OK": True, "Value": QUEUE_CONFIG} # check returned value @@ -158,7 +156,6 @@ class Test_getMQParamFromCSFailureTestCase(unittest.TestCase): """Test class to check known failure scenarios.""" def setUp(self): - # external dependencies module.CSAPI = MagicMock() module.gConfig = MagicMock() diff --git a/src/DIRAC/Resources/ProxyProvider/PUSPProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/PUSPProxyProvider.py index 2702521d471..f7827f1e033 100644 --- a/src/DIRAC/Resources/ProxyProvider/PUSPProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/PUSPProxyProvider.py @@ -16,7 +16,6 @@ class PUSPProxyProvider(ProxyProvider): def __init__(self, parameters=None): - super().__init__(parameters) def checkStatus(self, userDN): diff --git a/src/DIRAC/Resources/ProxyProvider/ProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/ProxyProvider.py index b26a02bdce9..e8a34de279a 100644 --- a/src/DIRAC/Resources/ProxyProvider/ProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/ProxyProvider.py @@ -5,7 +5,6 @@ class ProxyProvider: def __init__(self, parameters=None): - self.parameters = parameters self.name = None if parameters: diff --git a/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py b/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py index 8c3640237c6..84e237333d7 100644 --- a/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py +++ b/src/DIRAC/Resources/ProxyProvider/ProxyProviderFactory.py @@ -12,7 +12,6 @@ class ProxyProviderFactory: - ############################################################################# def __init__(self): """Standard constructor""" diff --git a/src/DIRAC/Resources/Storage/DIPStorage.py b/src/DIRAC/Resources/Storage/DIPStorage.py index 4cec45b1f14..346ccb3ad9b 100755 --- a/src/DIRAC/Resources/Storage/DIPStorage.py +++ b/src/DIRAC/Resources/Storage/DIPStorage.py @@ -23,7 +23,6 @@ class DIPStorage(StorageBase): - _INPUT_PROTOCOLS = ["file", "dip", "dips"] _OUTPUT_PROTOCOLS = ["dip", "dips"] diff --git a/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py b/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py index 93281a22e30..05cf1921e32 100644 --- a/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py +++ b/src/DIRAC/Resources/Storage/GFAL2_StorageBase.py @@ -733,7 +733,6 @@ def prestageFileStatus(self, path): failed = {} successful = {} for url, token in urls.items(): - try: successful[url] = self._prestageSingleFileStatus(url, token) except Exception as e: @@ -758,7 +757,6 @@ def _prestageSingleFileStatus(self, path: str, token: str) -> bool: token = str(token) with setGfalSetting(self.ctx, "BDII", "ENABLE", True): - # 0: not staged # 1: staged status = self.ctx.bring_online_poll(path, token) @@ -789,7 +787,6 @@ def releaseFile(self, path): successful = {} with setGfalSetting(self.ctx, "BDII", "ENABLE", True): for url, token in urls.items(): - # token could be an int token = str(token) try: @@ -1011,7 +1008,6 @@ def _listSingleDirectory(self, path: str, internalCall: bool = False) -> dict[st lfnStart = "/" for entry in listing: - nextEntry = dict(pathDict) nextEntry["FileName"] = os.path.join(pathDict["FileName"], entry) nextUrl = returnValueOrRaise(pfnunparse(nextEntry, srmSpecific=self.srmSpecificParse)) @@ -1238,7 +1234,6 @@ def _putSingleDirectory(self, src_directory: str, dest_directory: str) -> dict[s relDir = os.path.relpath(root, src_directory) for fileName in files: - # That is the full path of the file localy localFilePath = os.path.join(root, fileName) diff --git a/src/DIRAC/Resources/Storage/ProxyStorage.py b/src/DIRAC/Resources/Storage/ProxyStorage.py index 8156c46b7db..85f75756f10 100644 --- a/src/DIRAC/Resources/Storage/ProxyStorage.py +++ b/src/DIRAC/Resources/Storage/ProxyStorage.py @@ -10,12 +10,10 @@ class ProxyStorage(StorageBase): - _INPUT_PROTOCOLS = ["file", "dip", "dips"] _OUTPUT_PROTOCOLS = ["dip", "dips"] def __init__(self, storageName, parameters): - StorageBase.__init__(self, storageName, parameters) self.pluginName = "Proxy" self.url = "DataManagement/StorageElementProxy" @@ -66,7 +64,6 @@ def getFile(self, path, localPath=False): return S_OK(resDict) def putFile(self, path, sourceSize=0): - client = Client(url=self.url) if sourceSize: diff --git a/src/DIRAC/Resources/Storage/RFIOStorage.py b/src/DIRAC/Resources/Storage/RFIOStorage.py index 4de38a9fddc..cc44743ba64 100755 --- a/src/DIRAC/Resources/Storage/RFIOStorage.py +++ b/src/DIRAC/Resources/Storage/RFIOStorage.py @@ -14,12 +14,10 @@ class RFIOStorage(StorageBase): - _INPUT_PROTOCOLS = ["file", "rfio"] _OUTPUT_PROTOCOLS = ["rfio"] def __init__(self, storageName, parameters): - StorageBase.__init__(self, storageName, parameters) self.spaceToken = self.protocolParameters["SpaceToken"] diff --git a/src/DIRAC/Resources/Storage/S3Storage.py b/src/DIRAC/Resources/Storage/S3Storage.py index 7d12e549377..ecd66e1f764 100644 --- a/src/DIRAC/Resources/Storage/S3Storage.py +++ b/src/DIRAC/Resources/Storage/S3Storage.py @@ -43,7 +43,6 @@ def _extractKeyFromS3Path(meth): @functools.wraps(meth) def extractKey(self, urls, *args, **kwargs): - # If set to False, we are already working with keys, so # skip all the splitting extractKeys = kwargs.pop("extractKeys", True) @@ -95,7 +94,6 @@ class S3Storage(StorageBase): _OUTPUT_PROTOCOLS = ["file", "s3", "http", "https"] def __init__(self, storageName, parameters): - super().__init__(storageName, parameters) aws_access_key_id = parameters.get("Aws_access_key_id") @@ -479,7 +477,6 @@ def _presigned_putFile(self, urls, sourceSize=0): presignedResponses = res["Value"]["Successful"] for dest_url, presignedResponse in presignedResponses.items(): - src_file = urls[dest_url] try: diff --git a/src/DIRAC/Resources/Storage/StorageBase.py b/src/DIRAC/Resources/Storage/StorageBase.py index 6f22e8c8cff..0aef091039b 100755 --- a/src/DIRAC/Resources/Storage/StorageBase.py +++ b/src/DIRAC/Resources/Storage/StorageBase.py @@ -58,7 +58,6 @@ class StorageBase: DYNAMIC_OPTIONS = {} def __init__(self, name, parameterDict): - self.name = name self.pluginName = "" # This is set by the storageFactory and is the @@ -445,7 +444,6 @@ def getOccupancy(self, **kwargs): occupancyURL = res["Value"] try: - # download the file locally tmpDirName = tempfile.mkdtemp() res = returnSingleResult(self.getFile(occupancyURL, localPath=tmpDirName)) diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 525cc578934..f5790ca8f68 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -225,13 +225,11 @@ def __init__(self, name, protocolSections=None, vo=None, hideExceptions=False): self.turlProtocols = factoryDict["TurlProtocols"] for storage in self.storages.values(): - storage.setStorageElement(self) self.log = sLog.getSubLogger("SE[%s]" % self.name) if self.valid: - self.useCatalogURL = gConfig.getValue("/Resources/StorageElements/%s/UseCatalogURL" % self.name, False) self.log.debug("useCatalogURL: %s" % self.useCatalogURL) @@ -453,7 +451,6 @@ def getOccupancy(self, unit="MB", **kwargs): # Try all of the storages one by one for storage in filteredPlugins: - # The result of the plugin is always in B res = storage.getOccupancy(**kwargs) if res["OK"]: @@ -483,7 +480,6 @@ def checkOccupancy(self, occupancyDict, unit): mandatoryParams = {"Total", "Free"} # Make sure all the mandatory parameters are present if set(occupancyDict) & mandatoryParams != mandatoryParams: - msg = "Missing mandatory parameters %s" % str(mandatoryParams - set(occupancyDict)) log.error(msg) return S_ERROR(msg) @@ -817,7 +813,6 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols=None): successful = {} # Generate the URLs for lfn in lfns: - # Source URL first res = srcPlugin.constructURLFromLFN(lfn, withWSUrl=True) if not res["OK"]: @@ -997,7 +992,6 @@ def __isLocalSE(self): return S_OK(False) def __getFileCatalog(self): - if not self.__fileCatalog: self.__fileCatalog = FileCatalog(vo=self.vo) return self.__fileCatalog @@ -1391,7 +1385,6 @@ def addAccountingOperation(self, urlDict, startDate, elapsedTime, storageParamet accountingDict["TransferTotal"] = len(urlDict) accountingDict["FinalStatus"] = "Failed" else: - succ = callRes.get("Value", {}).get("Successful", {}) failed = callRes.get("Value", {}).get("Failed", {}) @@ -1427,7 +1420,6 @@ def addAccountingOperation(self, urlDict, startDate, elapsedTime, storageParamet # Send also the list of failures, only if we send to monitoring failedRecords = [] if "Monitoring" in self.dataOpSender.monitoringOptions: - for failedURL, errorMsg in failed.items(): failedRecord = { "timestamp": int(toEpochMilliSeconds()), diff --git a/src/DIRAC/Resources/Storage/Utilities.py b/src/DIRAC/Resources/Storage/Utilities.py index 441f9182516..374254ffb94 100644 --- a/src/DIRAC/Resources/Storage/Utilities.py +++ b/src/DIRAC/Resources/Storage/Utilities.py @@ -8,6 +8,8 @@ from DIRAC.Core.Utilities.ReturnValues import DReturnType T = TypeVar("T") + + # mypy doesn't understand default parameter values with generics so use overloads (python/mypy#3737) @overload def checkArgumentFormat(path: Union[str, list[str]]) -> DReturnType[dict[str, Literal[False]]]: diff --git a/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py b/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py index 84b1eef8f21..e46fdb01b6f 100755 --- a/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py +++ b/src/DIRAC/Resources/Storage/test/FIXME_Test_StorageElement.py @@ -321,7 +321,6 @@ def test_getFileSize(self): self.assertTrue(expectedError in directorySizeRes["Message"]) def test_getURL(self): - print( "\n\n#########################################################" "################\n\n\t\tGet access url test\n" diff --git a/src/DIRAC/Resources/Storage/test/FIXME_Test_StoragePlugIn.py b/src/DIRAC/Resources/Storage/test/FIXME_Test_StoragePlugIn.py index 2f5b4cdc348..1a24d48166b 100755 --- a/src/DIRAC/Resources/Storage/test/FIXME_Test_StoragePlugIn.py +++ b/src/DIRAC/Resources/Storage/test/FIXME_Test_StoragePlugIn.py @@ -29,7 +29,6 @@ class StoragePlugInTestCase(unittest.TestCase): """Base class for the StoragePlugin test cases""" def setUp(self): - factory = StorageFactory("lhcb") res = factory.getStorages(storageElementToTest, [plugin]) self.assertTrue(res["OK"]) diff --git a/src/DIRAC/Resources/Storage/test/Test_GFAL2_XROOTStorage.py b/src/DIRAC/Resources/Storage/test/Test_GFAL2_XROOTStorage.py index d48b2b4d4d2..8de3f9d11a5 100644 --- a/src/DIRAC/Resources/Storage/test/Test_GFAL2_XROOTStorage.py +++ b/src/DIRAC/Resources/Storage/test/Test_GFAL2_XROOTStorage.py @@ -25,7 +25,6 @@ def setUp(self): ) def test_constructURLFromLFN(self): - resource = GFAL2_XROOTStorage("storageName", self.parameterDict) resource.se = MagicMock() @@ -48,6 +47,5 @@ def test_constructURLFromLFN(self): if __name__ == "__main__": - suite = unittest.defaultTestLoader.loadTestsFromTestCase(XROOTStorage_TestCase) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageElement.py b/src/DIRAC/Resources/Storage/test/Test_StorageElement.py index 9c45af92652..d442392a69d 100644 --- a/src/DIRAC/Resources/Storage/test/Test_StorageElement.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageElement.py @@ -99,7 +99,6 @@ class TestBase(unittest.TestCase): "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None ) # Don't send accounting def setUp(self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation): - # Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), "test_StorageElement.cfg") cfgContent = """ @@ -692,7 +691,6 @@ class TestSameSE(unittest.TestCase): "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None ) # Don't send accounting def setUp(self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation): - # Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), "test_StorageElement.cfg") cfgContent = """ @@ -849,7 +847,6 @@ def test_01_compareSEWithItself(self, _mk_isLocalSE, _mk_addAccounting): self.storageB, self.storageBWithOtherBasePath, ): - self.assertTrue(se.isSameSE(se)) @mock.patch( diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py index 549bc2ede42..479359faab3 100755 --- a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py @@ -275,6 +275,7 @@ def loadCS(): def mock_StorageFactory__generateStorageObject(*args, **kwargs): """Don't really load the plugin, just create an object""" + # We create this FakeStorage object because if we just # return a plain object, we get a lot of AttributeErrors # later in the test diff --git a/src/DIRAC/Resources/scripts/dirac_resource_info.py b/src/DIRAC/Resources/scripts/dirac_resource_info.py index 04b4096ec0e..13ae3e2d1a3 100644 --- a/src/DIRAC/Resources/scripts/dirac_resource_info.py +++ b/src/DIRAC/Resources/scripts/dirac_resource_info.py @@ -8,7 +8,6 @@ @Script() def main(): - from DIRAC import S_OK, gLogger, gConfig, exit as DIRACExit ceFlag = False @@ -42,7 +41,6 @@ def setVOName(args): from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus def printCEInfo(voName): - resultQueues = Resources.getQueues(community=voName) if not resultQueues["OK"]: gLogger.error("Failed to get CE information") @@ -63,7 +61,6 @@ def printCEInfo(voName): siteStatus = "Active" if site in siteMaskList else "InActive" siteNew = True for ce in resultQueues["Value"][site]: - ceStatus = siteStatus if rssClient.rssFlag: result = rssClient.getElementStatus(ce, "ComputingElement") @@ -86,7 +83,6 @@ def printCEInfo(voName): return S_OK() def printSEInfo(voName): - fields = ("SE", "Status", "Protocols") records = [] diff --git a/src/DIRAC/StorageManagementSystem/Agent/RequestFinalizationAgent.py b/src/DIRAC/StorageManagementSystem/Agent/RequestFinalizationAgent.py index 8ce21d97422..ed598c9575b 100755 --- a/src/DIRAC/StorageManagementSystem/Agent/RequestFinalizationAgent.py +++ b/src/DIRAC/StorageManagementSystem/Agent/RequestFinalizationAgent.py @@ -9,7 +9,6 @@ class RequestFinalizationAgent(AgentModule): def initialize(self): - # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. diff --git a/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py b/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py index 97e3349c58f..96e8be89100 100755 --- a/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py +++ b/src/DIRAC/StorageManagementSystem/Agent/StageMonitorAgent.py @@ -37,7 +37,6 @@ def initialize(self): return S_OK() def execute(self): - res = getProxyInfo(disableVOMS=True) if not res["OK"]: return res diff --git a/src/DIRAC/StorageManagementSystem/Agent/StageRequestAgent.py b/src/DIRAC/StorageManagementSystem/Agent/StageRequestAgent.py index 160af77187b..4de4ad78785 100755 --- a/src/DIRAC/StorageManagementSystem/Agent/StageRequestAgent.py +++ b/src/DIRAC/StorageManagementSystem/Agent/StageRequestAgent.py @@ -25,7 +25,6 @@ def initialize(self): return S_OK() def execute(self): - # Get the current submitted stage space and the amount of pinned space for each storage element res = self.getStorageUsage() if not res["OK"]: @@ -305,7 +304,6 @@ def _issuePrestageRequests(self, storageElement, seReplicaIDs, allReplicaInfo): return def __sortBySE(self, replicaDict): - seReplicas = {} replicaIDs = {} for replicaID, info in replicaDict.items(): @@ -469,7 +467,6 @@ def __checkIntegrity(self, storageElement, seReplicaIDs, allReplicaInfo): onlineReplicaIDs = [] offlineReplicaIDs = [] for lfn, metadata in res["Value"]["Successful"].items(): - if metadata["Size"] != allReplicaInfo[lfnRepIDs[lfn]]["Size"]: gLogger.error("StageRequest.__checkIntegrity: LFN StorageElement size does not match FileCatalog", lfn) terminalReplicaIDs[lfnRepIDs[lfn]] = "LFN StorageElement size does not match FileCatalog" diff --git a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py index df22bda200d..6e9a90e6012 100644 --- a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -641,7 +641,6 @@ def __removeWMSTasks(self, transJobIDs): jobIDs = [int(j) for j in transJobIDs if int(j)] allRemove = True for jobList in breakListIntoChunks(jobIDs, 500): - res = self.wmsClient.killJob(jobList) if res["OK"]: self.log.info("Successfully killed %d jobs from WMS" % len(jobList)) diff --git a/src/DIRAC/TransformationSystem/Client/RequestTasks.py b/src/DIRAC/TransformationSystem/Client/RequestTasks.py index 46de7472fb0..b2912640ef7 100644 --- a/src/DIRAC/TransformationSystem/Client/RequestTasks.py +++ b/src/DIRAC/TransformationSystem/Client/RequestTasks.py @@ -191,7 +191,6 @@ def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): - transID = task["TransformationID"] oRequest = Request() diff --git a/src/DIRAC/TransformationSystem/Client/Transformation.py b/src/DIRAC/TransformationSystem/Client/Transformation.py index 7e3967eb006..d964a9025f6 100644 --- a/src/DIRAC/TransformationSystem/Client/Transformation.py +++ b/src/DIRAC/TransformationSystem/Client/Transformation.py @@ -18,7 +18,6 @@ class Transformation(API): - ############################################################################# def __init__(self, transID=0, transClient=None): """c'tor""" diff --git a/src/DIRAC/TransformationSystem/Client/test/Test_Client_RequestTasks.py b/src/DIRAC/TransformationSystem/Client/test/Test_Client_RequestTasks.py index fe7b9be7847..a6f7b77d8d5 100644 --- a/src/DIRAC/TransformationSystem/Client/test/Test_Client_RequestTasks.py +++ b/src/DIRAC/TransformationSystem/Client/test/Test_Client_RequestTasks.py @@ -91,7 +91,6 @@ def test_prepareSingleOperationsBody(transBody, owner, taskDict): assert len(res["Value"]) == originalNbOfTasks for _taskID, task in taskDict.items(): - req = task.get("TaskObject") # Checks whether we got a Request assigned @@ -178,7 +177,6 @@ def test_prepareMultiOperationsBody(transBody, owner, taskDict): assert len(res["Value"]) == originalNbOfTasks for _taskID, task in taskDict.items(): - req = task.get("TaskObject") # Checks whether we got a Request assigned @@ -195,7 +193,6 @@ def test_prepareMultiOperationsBody(transBody, owner, taskDict): # Loop over each operation # to check their attributes for opsID, ops in enumerate(req): - expectedOpsType, expectedOpsAttributes = transBody[opsID] # Compare the operation type with what we want @@ -203,7 +200,6 @@ def test_prepareMultiOperationsBody(transBody, owner, taskDict): # Check the operation attributes one after the other for opsAttr, opsVal in expectedOpsAttributes.items(): - # If the expected value starts with 'TASK:' # we should make the substitution with whatever is in # the taskDict. @@ -267,7 +263,6 @@ def test_prepareProblematicMultiOperationsBody(transBody, owner, taskDict): # I just prepare the future :-) for _taskID, task in taskDict.items(): - req = task.get("TaskObject") # Checks whether we got a Request assigned @@ -284,7 +279,6 @@ def test_prepareProblematicMultiOperationsBody(transBody, owner, taskDict): # Loop over each operation # to check their attributes for opsID, ops in enumerate(req): - expectedOpsType, expectedOpsAttributes = transBody[opsID] # Compare the operation type with what we want @@ -292,7 +286,6 @@ def test_prepareProblematicMultiOperationsBody(transBody, owner, taskDict): # Check the operation attributes one after the other for opsAttr, opsVal in expectedOpsAttributes.items(): - # If the expected value starts with 'TASK:' # we should make the substitution with whatever is in # the taskDict. @@ -340,7 +333,6 @@ def test_complexBodyPlugin(taskDict, pluginFactor): assert len(res["Value"]) == originalNbOfTasks for _taskID, task in taskDict.items(): - req = task.get("TaskObject") # Checks whether we got a Request assigned diff --git a/src/DIRAC/TransformationSystem/Client/test/Test_Client_TaskManagerPlugin.py b/src/DIRAC/TransformationSystem/Client/test/Test_Client_TaskManagerPlugin.py index c462009e950..4dee83d8d36 100644 --- a/src/DIRAC/TransformationSystem/Client/test/Test_Client_TaskManagerPlugin.py +++ b/src/DIRAC/TransformationSystem/Client/test/Test_Client_TaskManagerPlugin.py @@ -131,7 +131,6 @@ class ClientsTestCase(unittest.TestCase): """Base class for the clients test cases""" def setUp(self): - gLogger.setLevel("DEBUG") self.mockTransClient = MagicMock() @@ -164,7 +163,6 @@ def tearDown(self): class TaskManagerPluginSuccess(ClientsTestCase): def test__BySE(self): - ourPG = importlib.import_module("DIRAC.TransformationSystem.Client.TaskManagerPlugin") ourPG.getSitesForSE = getSitesForSE p_o = TaskManagerPlugin("BySE", operationsHelper=MagicMock()) @@ -202,7 +200,6 @@ def test__BySE(self): self.assertEqual(res, {"CERN", "CSCS"}) def test__ByJobType(self): - ourPG = importlib.import_module("DIRAC.TransformationSystem.Client.TaskManagerPlugin") ourPG.getSites = getSitesFake ourPG.getSitesForSE = getSitesForSE diff --git a/src/DIRAC/TransformationSystem/Client/test/Test_Client_TransformationSystem.py b/src/DIRAC/TransformationSystem/Client/test/Test_Client_TransformationSystem.py index c158cd982a3..5a40114f84a 100644 --- a/src/DIRAC/TransformationSystem/Client/test/Test_Client_TransformationSystem.py +++ b/src/DIRAC/TransformationSystem/Client/test/Test_Client_TransformationSystem.py @@ -107,7 +107,6 @@ def test_groupByReplicas(self): class RequestTasksSuccess(ClientsTestCase): def test_prepareTranformationTasks(self): - # No tasks in input taskDict = {} res = self.requestTasks.prepareTransformationTasks("", taskDict, "owner", "ownerGroup", "/bih/boh/DN") @@ -207,7 +206,6 @@ def test_prepareTranformationTasks(self): class TransformationSuccess(ClientsTestCase): def test_setGet(self): - res = self.transformation.setTransformationName("TestTName") self.assertTrue(res["OK"]) description = "Test transformation description" diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.py b/src/DIRAC/TransformationSystem/DB/TransformationDB.py index 24e818e70f3..b6084adcc22 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.py +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.py @@ -763,7 +763,6 @@ def __insertExistingTransformationFiles(self, transID, fileTuplesList, connectio # splitting in various chunks, in case it is too big for fileTuples in breakListIntoChunks(fileTuplesList, 10000): - gLogger.verbose( "Adding first %d files in TransformationFiles (out of %d)" % (len(fileTuples), len(fileTuplesList)) ) diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_information.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_information.py index b756844d1d6..cfe75efbb80 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_information.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_information.py @@ -12,7 +12,6 @@ @Script() def main(): - informations = [ "AuthorDN", "AuthorGroup", diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_update_derived.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_update_derived.py index 67330998b23..245fd90f836 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_update_derived.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_update_derived.py @@ -11,7 +11,6 @@ @Script() def main(): - Script.registerSwitch("", "NoReset", "Don't reset the MaxReset files to Unused (default is to reset)") Script.parseCommandLine(ignoreErrors=True) diff --git a/src/DIRAC/Workflow/Modules/FailoverRequest.py b/src/DIRAC/Workflow/Modules/FailoverRequest.py index 4b5b9030d83..604e653011c 100644 --- a/src/DIRAC/Workflow/Modules/FailoverRequest.py +++ b/src/DIRAC/Workflow/Modules/FailoverRequest.py @@ -12,7 +12,6 @@ class FailoverRequest(ModuleBase): - ############################################################################# def __init__(self): @@ -38,7 +37,6 @@ def _initialize(self): self.request.SourceComponent = "Job_%d" % self.jobID def _execute(self): - # report on the status of the input data, by default they are 'Processed', unless the job failed # failures happening before are not touched filesInFileReport = self.fileReport.getFiles() diff --git a/src/DIRAC/Workflow/Modules/UploadOutputs.py b/src/DIRAC/Workflow/Modules/UploadOutputs.py index f838324929c..109d60a3e61 100644 --- a/src/DIRAC/Workflow/Modules/UploadOutputs.py +++ b/src/DIRAC/Workflow/Modules/UploadOutputs.py @@ -10,7 +10,6 @@ class UploadOutputs(ModuleBase): - ############################################################################# def __init__(self): diff --git a/src/DIRAC/Workflow/Modules/test/Test_Modules.py b/src/DIRAC/Workflow/Modules/test/Test_Modules.py index 6eac1a37c9e..9e5745ef010 100644 --- a/src/DIRAC/Workflow/Modules/test/Test_Modules.py +++ b/src/DIRAC/Workflow/Modules/test/Test_Modules.py @@ -14,7 +14,6 @@ class ModulesTestCase(unittest.TestCase): """Base class for the Modules test cases""" def setUp(self): - gLogger.setLevel("ERROR") # import sys # sys.modules["DIRAC"] = DIRAC.ResourceStatusSystem.test.fake_Logger @@ -445,17 +444,14 @@ def tearDown(self): class ModuleBaseSuccess(ModulesTestCase): - ################################################# def test__checkLocalExistance(self): - self.assertRaises(OSError, self.mb._checkLocalExistance, ["aaa", "bbb"]) ################################################# def test__applyMask(self): - candidateFiles = { "00012345_00012345_4.dst": { "lfn": "/lhcb/MC/2010/DST/00012345/0001/00012345_00012345_4.dst", @@ -508,7 +504,6 @@ def test__applyMask(self): ################################################# def test__checkSanity(self): - candidateFiles = { "00012345_00012345_4.dst": { "lfn": "/lhcb/MC/2010/DST/00012345/0001/00012345_00012345_4.dst", @@ -627,7 +622,6 @@ def test_getCandidateFiles(self): self.assertEqual(res, result) def test__enableModule(self): - self.mb.production_id = self.prod_id self.mb.prod_job_id = self.prod_job_id self.mb.jobID = self.wms_job_id @@ -645,7 +639,6 @@ def test__enableModule(self): self.assertTrue(self.mb._enableModule()) def test__determineStepInputData(self): - self.mb.stepName = "DaVinci_2" inputData = "previousStep" @@ -716,11 +709,9 @@ def test__determineStepInputData(self): class FailoverRequestSuccess(ModulesTestCase): - ################################################# def test_execute(self): - self.fr.jobType = "merge" self.fr.stepInputData = ["foo", "bar"] @@ -749,11 +740,9 @@ def test_execute(self): class ScriptSuccess(ModulesTestCase): - ################################################# def test_execute(self): - self.script.jobType = "merge" self.script.stepInputData = ["foo", "bar"] @@ -779,11 +768,9 @@ def test_execute(self): class ScriptUnicode(ModulesTestCase): - ################################################# def test_execute(self): - self.script.jobType = "merge" self.script.stepInputData = ["foo", "bar"] @@ -810,11 +797,9 @@ def test_execute(self): class ScriptFailure(ModulesTestCase): - ################################################# def test_execute(self): - self.script.jobType = "merge" self.script.stepInputData = ["foo", "bar"] diff --git a/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py b/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py index 5d9fb1441c6..582810f12a2 100644 --- a/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py +++ b/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py @@ -12,7 +12,6 @@ class UtilitiesTestCase(unittest.TestCase): """Base class""" def setUp(self): - self.job = Job() pass diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/CloudDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/CloudDirector.py index 16600d552f9..068453c4105 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/CloudDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/CloudDirector.py @@ -57,7 +57,6 @@ def initialize(self): return S_OK() def beginExecution(self): - # The Director is for a particular user community self.vo = self.am_getOption("VO", "") if not self.vo: @@ -500,7 +499,6 @@ def createVMs(self): return S_OK() def getVMInstances(self, endpoint, maxInstances): - result = virtualMachineDB.getInstanceCounters("Status", {"Endpoint": endpoint}) if not result["OK"]: return result diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py index 99540a74ff2..685b648415f 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py @@ -140,7 +140,6 @@ def execute(self): return result for queueName, queueDictionary in queueDictItems: - # Make sure there is no problem with the queue before trying to submit if not self._allowedToSubmit(queueName): continue @@ -180,7 +179,6 @@ def execute(self): # Try to match a job jobRequest = self._matchAJob(ceDictList) while jobRequest["OK"]: - # Check matcher information returned matcherParams = ["JDL", "DN", "Group"] matcherInfo = jobRequest["Value"] diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 290c47f31f8..797a5ff3b3a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -27,6 +27,7 @@ from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.TimeUtilities import second, toEpochMilliSeconds from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager +from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus @@ -214,6 +215,10 @@ def beginExecution(self): if cesOption and "any" not in [ce.lower() for ce in cesOption]: ces = cesOption + tags = self.am_getOption("Tags", []) + if not tags: + tags = None + self.log.always("VO:", self.vo) if self.voGroups: self.log.always("Group(s):", self.voGroups) @@ -223,7 +228,9 @@ def beginExecution(self): self.log.always("PilotDN:", self.pilotDN) self.log.always("PilotGroup:", self.pilotGroup) - result = self.resourcesModule.getQueues(community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes) + result = self.resourcesModule.getQueues( + community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, tags=tags + ) if not result["OK"]: return result result = getQueuesResolved( @@ -240,7 +247,6 @@ def beginExecution(self): self.queueDict = result["Value"] for __queueName, queueDict in self.queueDict.items(): - # Update self.sites if queueDict["Site"] not in self.sites: self.sites.append(queueDict["Site"]) @@ -435,6 +441,13 @@ def submitPilots(self): lifetime_secs = result["Value"] ce.setProxy(proxy, lifetime_secs) + # Get valid token if needed + if "Token" in ce.ceParameters.get("Tag", []): + result = self.__getPilotToken() + if not result["OK"]: + return result + ce.setToken(result["Value"], 3500) + # now really submitting res = self._submitPilotsToQueue(pilotsToSubmit, ce, queueName) if not res["OK"]: @@ -450,6 +463,23 @@ def submitPilots(self): return S_OK() + def __getPilotToken(self): + """Get the token corresponding to the pilot user identity + + :return: S_OK/S_ERROR, Token object as Value + """ + + result = Registry.getUsernameForDN(self.pilotDN) + if not result["OK"]: + return result + userName = result["Value"] + result = gTokenManager.getToken( + userName=userName, + userGroup=self.pilotGroup, + requiredTimeLeft=3600, + ) + return result + def _ifAndWhereToSubmit(self): """Return a tuple that says if and where to submit pilots: @@ -855,7 +885,6 @@ def getQueueSlots(self, queue, manyWaitingPilotsFlag): waitingJobs = 1 if totalSlots == 0: if availableSlotsCount % self.availableSlotsUpdateCycleFactor == 0: - # Get the list of already existing pilots for this queue jobIDList = None result = pilotAgentsDB.selectPilots( @@ -1206,6 +1235,13 @@ def _updatePilotStatusPerQueue(self, queue, proxy): if not result["OK"]: ce.setProxy(proxy, 23300) + # Get valid token if needed + if "Token" in ce.ceParameters.get("Tag", []): + result = self.__getPilotToken() + if not result["OK"]: + return result + ce.setToken(result["Value"], 3500) + result = ce.getJobStatus(stampedPilotRefs) if not result["OK"]: self.log.error("Failed to get pilots status from CE", f"{ceName}: {result['Message']}") diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py index dca68facd1d..72784279565 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py @@ -32,7 +32,6 @@ def __getCSConfig(self): setattr(self, varName, value) for csOption, csDefault, varName in (("JobWrappersLocation", "/scratch", "vmJobWrappersLocation"),): - path = f"{imgPath}/{csOption}" value = gConfig.getValue(path, csDefault) if not value: @@ -74,7 +73,6 @@ def __declareInstanceRunning(self): return S_ERROR("Could not declare instance running after %d retries" % retries) def initialize(self): - self.op = Operations.Operations() # Init vars self.runningPod = gConfig.getValue("/LocalSite/RunningPod") @@ -153,7 +151,6 @@ def __getNumJobWrappers(self): return len(jdlList) def execute(self): - # Get load avgLoad, avgRequiredSamples = self.__getLoadAvg() self.log.info("Load Average", f"{avgLoad:.2f}") diff --git a/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py b/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py index e1a39a372cf..ed694fa9f93 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py @@ -15,7 +15,6 @@ class InputDataByProtocol: - ############################################################################# def __init__(self, argumentsDict): """Standard constructor""" diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py index c8b08f380fd..288b89143fe 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py @@ -8,7 +8,6 @@ @createClient("WorkloadManagement/JobMonitoring") class JobMonitoringClient(Client): def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("WorkloadManagement/JobMonitoring") diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobReport.py b/src/DIRAC/WorkloadManagementSystem/Client/JobReport.py index 67285044083..6af63f6d1c1 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobReport.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobReport.py @@ -149,7 +149,6 @@ def generateForwardDISET(self): if not result["OK"]: gLogger.error("Error while sending the job status", result["Message"]) if "rpcStub" in result: - rpcStub = result["rpcStub"] forwardDISETOp = Operation() diff --git a/src/DIRAC/WorkloadManagementSystem/Client/Limiter.py b/src/DIRAC/WorkloadManagementSystem/Client/Limiter.py index abd7c7797a9..a532eeaf2e7 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/Limiter.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/Limiter.py @@ -13,7 +13,6 @@ class Limiter: - # static variables shared between all instances of this class csDictCache = DictCache() condCache = DictCache() diff --git a/src/DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py b/src/DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py index 2a16f5e3e58..c03b85e8d49 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py @@ -12,7 +12,6 @@ class PoolXMLSlice: - ############################################################################# def __init__(self, catalogName): """Standard constructor""" diff --git a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py index 557243c04cb..76316b241b6 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py @@ -21,7 +21,6 @@ class SandboxStoreClient: - __validSandboxTypes = ("Input", "Output") __smdb = None diff --git a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py index cc66eb29fa9..7fef299f71f 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py @@ -33,7 +33,6 @@ @pytest.fixture def setUp(): - yield setUp try: @@ -44,7 +43,6 @@ def setUp(): def test__processResourceDescription(setUp): - resourceDescription = { "Architecture": "x86_64-slc6", "CEQueue": "jenkins-queue_not_important", @@ -84,7 +82,6 @@ def test__processResourceDescription(setUp): def test_uploadFilesAsSandbox(mocker, setUp): - mocker.patch("DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient.TransferClient", return_value=MagicMock()) ssc = SandboxStoreClient() fileList = [BytesIO(b"try")] diff --git a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg index 3bc072d0746..85cedcf0715 100644 --- a/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg @@ -279,6 +279,8 @@ Agents CETypes = any # List of CEs that will be treated by this SiteDirector ("any" can refer to any type of CE defined in the CS) CEs = any + # List of Tags that are required to be present in the CE/Queue definition + Tags = # The maximum length of a queue (in seconds). Default: 3 days MaxQueueLength = 259200 diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index 0bbbe1ecf5e..251b6127d6c 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -1207,7 +1207,6 @@ def removeJobFromDB(self, jobIDs): "Jobs", "JobJDLs", ]: - cmd = "DELETE FROM {} WHERE JobID in ({})".format(table, ",".join(str(j) for j in jobIDList)) result = self._update(cmd) if not result["OK"]: @@ -1475,7 +1474,6 @@ def getSiteMaskStatus(self, sites=None): return S_OK(dict(result["Value"])) elif isinstance(sites, str): - ret = self._escapeString(sites) if not ret["OK"]: return ret diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py index 33f6f28d344..87bd34a6e12 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -34,7 +34,6 @@ class PilotAgentsDB(DB): def __init__(self, parentLogger=None): - super().__init__("PilotAgentsDB", "WorkloadManagement/PilotAgentsDB", parentLogger=parentLogger) self.lock = threading.Lock() @@ -1272,5 +1271,4 @@ def buildSQL(self, selectDict=None): return finalQuery def getColumnList(self): - return self._columns diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py index a5c5292b357..bfaf211409f 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py @@ -31,7 +31,6 @@ ############################################################################# class PilotsLoggingDB: def __init__(self, parentLogger=None): - if not parentLogger: parentLogger = gLogger self.log = parentLogger.getSubLogger(self.__class__.__name__) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.py b/src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.py index acb52430cdc..e7b860158c6 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.py @@ -33,7 +33,6 @@ class VirtualMachineDB(DB): - # When checking the Status on the DB it must be one of these values, if not, the last one (Error) is set # When declaring a new Status, it will be set to Error if not in the list validImageStates = ["New", "Validated", "Error"] @@ -419,9 +418,7 @@ def getInstancesByStatus(self, status): tableName, _validStates, idName = self.__getTypeTuple("Image") for imageID, uniqueID in runningInstances: - if imageID not in imagesDict: - imageName = self.getFields(tableName, ["Name"], {idName: imageID}) if not imageName["OK"]: continue @@ -704,7 +701,6 @@ def getHistoryValues(self, averageBucket, selDict=None, fields2Get=False, timesp return S_OK({"ParameterNames": paramFields, "Records": finalData}) def getRunningInstancesHistory(self, timespan=0, bucketSize=900): - try: bucketSize = max(300, int(bucketSize)) except ValueError: @@ -1056,7 +1052,6 @@ def __getImageID(self, imageName): ) if ret["OK"] and "lastRowId" in ret: - rowID = ret["lastRowId"] ret = self.getFields(tableName, [idName], {"Name": imageName}) diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/InputData.py b/src/DIRAC/WorkloadManagementSystem/Executor/InputData.py index 381ea5c93e1..e71486c1142 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/InputData.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/InputData.py @@ -24,7 +24,6 @@ class InputData(OptimizerExecutor): """ def __init__(self) -> None: - self.__lastCacheUpdate = 0 self.__cacheLifeTime = 600 super().__init__() diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py index 5f06ddcd8bc..2b748ee72ce 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py @@ -95,7 +95,6 @@ def optimizeJob(self, jid, jobState): # If the user has selected any site, filter them and hold the job if not able to run if userSites: if jobType not in self.ex_getOption("ExcludedOnHoldJobTypes", []): - result = self.siteClient.getUsableSites(userSites) if not result["OK"]: self.jobLog.error( @@ -481,7 +480,6 @@ def __resolveStaging(self, inputData, idSites): return (True, bestSites) def __preRequestStaging(self, jobManifest, stageSite, opData): - tapeSEs = [] diskSEs = [] vo = jobManifest.getOption("VirtualOrganization") diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/test/Test_Executor.py b/src/DIRAC/WorkloadManagementSystem/Executor/test/Test_Executor.py index 58787e72817..2f5d56501fb 100644 --- a/src/DIRAC/WorkloadManagementSystem/Executor/test/Test_Executor.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/test/Test_Executor.py @@ -98,7 +98,6 @@ def test__getTagsFromManifest(manifestOptions, expected): ], ) def test__getInputSandbox(mocker, manifestOptions, expected): - mocker.patch("DIRAC.WorkloadManagementSystem.Client.JobState.JobState.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Client.JobState.JobState.JobLoggingDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Client.JobState.JobState.TaskQueueDB.__init__", side_effect=mockNone) @@ -173,7 +172,6 @@ def test__getInputSandbox(mocker, manifestOptions, expected): ], ) def test__getSiteCandidates(mocker, okReplicas, getSitesForSE_RV, storageGetStatus_RV, expectedRes, expectedValue): - mockSE = MagicMock() mockSE.getStatus.return_value = storageGetStatus_RV diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 48e0a51a803..d03f03a1903 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -830,7 +830,6 @@ def processJobOutputs(self): self.log.error("Output sandbox upload failed with message", result_sbUpload["Message"]) outputSandboxData = result_sbUpload.get("SandboxFileName") if outputSandboxData: - self.log.info("Attempting to upload %s as output data" % (outputSandboxData)) if self.failedFlag: outputData = [outputSandboxData] @@ -1457,7 +1456,6 @@ def __setJobParamList(self, value, sendFlag=False): class ExecutionThread(threading.Thread): - ############################################################################# def __init__(self, spObject, cmd, maxPeekLines, stdoutFile, stderrFile, exeEnv): threading.Thread.__init__(self) @@ -1533,7 +1531,6 @@ def rescheduleFailedJob(jobID, minorStatus, jobReport=None): rescheduleResult = JobStatus.RESCHEDULED try: - gLogger.warn("Failure during", minorStatus) # Setting a job parameter does not help since the job will be rescheduled, diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py index e79d8738baf..382c76954cd 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py @@ -39,7 +39,6 @@ class Watchdog: - ############################################################################# def __init__(self, pid, exeThread, spObject, jobCPUTime, memoryLimit=0, processors=1, jobArgs={}): """Constructor, takes system flag as argument.""" diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py index 01ae809748c..6f3849c162c 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobMonitoringHandler.py @@ -219,7 +219,6 @@ def export_getCounters(cls, attrList, attrDict=None, cutDate=""): @classmethod def export_getJobOwner(cls, jobID): - return cls.jobDB.getJobAttribute(jobID, "Owner") ############################################################################## @@ -227,7 +226,6 @@ def export_getJobOwner(cls, jobID): @classmethod def export_getJobSite(cls, jobID): - return cls.jobDB.getJobAttribute(jobID, "Site") ############################################################################## @@ -235,7 +233,6 @@ def export_getJobSite(cls, jobID): @classmethod def export_getJobJDL(cls, jobID, original): - return cls.jobDB.getJobJDL(jobID, original=original) ############################################################################## @@ -243,7 +240,6 @@ def export_getJobJDL(cls, jobID, original): @classmethod def export_getJobLoggingInfo(cls, jobID): - return cls.jobLoggingDB.getJobLoggingInfo(jobID) ############################################################################## diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobPolicy.py b/src/DIRAC/WorkloadManagementSystem/Service/JobPolicy.py index e55579495ad..efa978c6120 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobPolicy.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobPolicy.py @@ -74,7 +74,6 @@ class JobPolicy: def __init__(self, userDN, userGroup, allInfo=True): - self.userDN = userDN self.userName = "" result = getUsernameForDN(userDN) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py index f293960d8e3..c359096bd27 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py @@ -358,7 +358,6 @@ def export_setJobsParameter(cls, jobsParameterDict): failed = False for jobID in jobsParameterDict: - if cls.elasticJobParametersDB: res = cls.elasticJobParametersDB.setJobParameter( int(jobID), str(jobsParameterDict[jobID][0]), str(jobsParameterDict[jobID][1]) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/OptimizationMindHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/OptimizationMindHandler.py index 276958e6c3c..fec748b9408 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/OptimizationMindHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/OptimizationMindHandler.py @@ -37,7 +37,6 @@ def cleanTaskQueues(): class OptimizationMindHandler(ExecutorMindHandler): - __optimizationStates = [JobStatus.RECEIVED, JobStatus.CHECKING] __loadTaskId = False diff --git a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py index 46cb769f71e..7847efa208a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py @@ -477,7 +477,6 @@ def export_getPilotStatistics(cls, attribute, selectDict): @classmethod def export_deletePilots(cls, pilotIDs): - if isinstance(pilotIDs, str): return cls.pilotAgentsDB.deletePilot(pilotIDs) @@ -508,7 +507,6 @@ def export_deletePilots(cls, pilotIDs): @classmethod def export_clearPilots(cls, interval=30, aborted_interval=7): - result = cls.pilotAgentsDB.clearPilots(interval, aborted_interval) if not result["OK"]: return result diff --git a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py index 888630cbe79..d1705f60cc5 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -27,7 +27,6 @@ class SandboxStoreHandler(RequestHandler): - __purgeCount = -1 __purgeLock = threading.Lock() __purgeWorking = False @@ -499,7 +498,6 @@ def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): if self.getCSOption("DelayedExternalDeletion", True): gLogger.info("Setting deletion request") try: - # We need the hostDN used in order to pass these credentials to the # SandboxStoreDB.. hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation() diff --git a/src/DIRAC/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py index 77edc1ec977..1411f538d14 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py @@ -59,7 +59,6 @@ def haltStalledInstances(cls): @classmethod def getCEInstances(cls, siteList=None, ceList=None, vo=None): - result = getVMTypes(siteList=siteList, ceList=ceList, vo=vo) if not result["OK"]: return result @@ -106,7 +105,6 @@ def checkStalledInstances(cls): @classmethod def stopInstance(cls, site, endpoint, nodeID): - result = getVMTypeConfig(site, endpoint) if not result["OK"]: return result @@ -121,7 +119,6 @@ def stopInstance(cls, site, endpoint, nodeID): @classmethod def createEndpoint(cls, uniqueID): - result = cls.virtualMachineDB.getEndpointFromInstance(uniqueID) if not result["OK"]: return result @@ -218,7 +215,6 @@ def initialize(self): types_getCEInstances = [(list, type(None)), (list, type(None)), str] def export_getCEInstances(self, siteList, ceList, vo): - if not siteList: siteList = None return self.getCEInstances(siteList=siteList, ceList=ceList, vo=vo) @@ -226,13 +222,11 @@ def export_getCEInstances(self, siteList, ceList, vo): types_stopInstance = [str, str, str] def export_stopInstance(self, site, endpoint, nodeID): - return self.stopInstance(site, endpoint, nodeID) types_getPilotOutput = [str] def export_getPilotOutput(self, pilotReference): - return self.getPilotOutput(pilotReference) types_checkVmWebOperation = [str] diff --git a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py index ac57d4fc6b9..87339bcbbd8 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py @@ -20,7 +20,6 @@ def getGridEnv(): - gridEnv = "" setup = gConfig.getValue("/DIRAC/Setup", "") if setup: diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py b/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py index daec5d876a1..2440ef97960 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py @@ -12,7 +12,6 @@ def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1): - if parList: if nPar != len(parList): return [] @@ -55,7 +54,6 @@ def getParameterVectorLength(jobClassAd): def __updateAttribute(classAd, attribute, parName, parValue): - # If there is something to do: pattern = r"%%\(%s\)s" % parName if parName == "0": diff --git a/src/DIRAC/WorkloadManagementSystem/private/correctors/BaseHistoryCorrector.py b/src/DIRAC/WorkloadManagementSystem/private/correctors/BaseHistoryCorrector.py index fe842aaeaf6..33726c30647 100644 --- a/src/DIRAC/WorkloadManagementSystem/private/correctors/BaseHistoryCorrector.py +++ b/src/DIRAC/WorkloadManagementSystem/private/correctors/BaseHistoryCorrector.py @@ -9,7 +9,6 @@ class BaseHistoryCorrector(BaseCorrector): - _GLOBAL_MAX_CORRECTION = "MaxGlobalCorrection" _SLICE_TIME_SPAN = "TimeSpan" _SLICE_WEIGHT = "Weight" @@ -80,7 +79,6 @@ def _getHistoryData(self, _timeSpan, _groupToUse): return S_ERROR("Not implemented !") def _getUsageHistoryForTimeSpan(self, timeSpan, groupToUse=""): - result = self._getHistoryData(timeSpan, groupToUse) if not result["OK"]: diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py index b44de00b466..23b5aa612ba 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_wms_pilot_job_info.py @@ -37,7 +37,6 @@ def main(): errorList = [] for gridID in args: - result = {} res = diracAdmin.getPilotInfo(gridID) if not res["OK"]: diff --git a/src/DIRAC/tests/Workflow/Integration/exe-script.py b/src/DIRAC/tests/Workflow/Integration/exe-script.py index 7fa418e3f5d..0eba06df21e 100644 --- a/src/DIRAC/tests/Workflow/Integration/exe-script.py +++ b/src/DIRAC/tests/Workflow/Integration/exe-script.py @@ -6,5 +6,4 @@ # Main if __name__ == "__main__": - sys.exit(subprocess.call(shlex.split("echo 'Hello World'"))) diff --git a/src/DIRAC/tests/Workflow/Integration/helloWorld.py b/src/DIRAC/tests/Workflow/Integration/helloWorld.py index 22693906087..5c241d5d1ae 100644 --- a/src/DIRAC/tests/Workflow/Integration/helloWorld.py +++ b/src/DIRAC/tests/Workflow/Integration/helloWorld.py @@ -6,5 +6,4 @@ # Main if __name__ == "__main__": - sys.exit(subprocess.call(shlex.split("echo Hello World"))) diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py b/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py index ca7e2a33b10..67ebf6431e7 100644 --- a/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py +++ b/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py @@ -128,7 +128,6 @@ def getRequestTimeAndCount(data, time): for result in results: i = 0 try: - # Ignore past while int(result[i][2]) < time: i += 1 diff --git a/tests/Integration/AccountingSystem/Test_DataStoreClient.py b/tests/Integration/AccountingSystem/Test_DataStoreClient.py index 98f9b89ac82..3620ce43a8c 100644 --- a/tests/Integration/AccountingSystem/Test_DataStoreClient.py +++ b/tests/Integration/AccountingSystem/Test_DataStoreClient.py @@ -20,7 +20,6 @@ def test_addAndRemoveStorageOccupancy(): - # just inserting one record record = createStorageOccupancyAccountingRecord() record.setStartTime() diff --git a/tests/Integration/AccountingSystem/Test_ReportsClient.py b/tests/Integration/AccountingSystem/Test_ReportsClient.py index c4509e4a44b..2e76c0b2bc2 100644 --- a/tests/Integration/AccountingSystem/Test_ReportsClient.py +++ b/tests/Integration/AccountingSystem/Test_ReportsClient.py @@ -25,7 +25,6 @@ def test_addAndRemoveStorageOccupancy(): - # just inserting one record record = createStorageOccupancyAccountingRecord() record.setStartTime() diff --git a/tests/Integration/Core/Test_ElasticsearchDB.py b/tests/Integration/Core/Test_ElasticsearchDB.py index f70a7e40867..b89938c7a60 100644 --- a/tests/Integration/Core/Test_ElasticsearchDB.py +++ b/tests/Integration/Core/Test_ElasticsearchDB.py @@ -441,6 +441,7 @@ def test_Search(): # [{u'total_quantity': {u'value': 5.0}, u'key': u'a', u'doc_count': 5}, { # u'total_quantity': {u'value': 8.0}, u'key': u'b', u'doc_count': 5}]) + # def test_piplineaggregation(): # q = [elasticSearchDB._Q('range', timestamp={'lte': 1423631917911, 'gte': 1423399451544})] # s = elasticSearchDB._Search(index_name) diff --git a/tests/Integration/DataManagementSystem/FIXME_dfc_dir_completion.py b/tests/Integration/DataManagementSystem/FIXME_dfc_dir_completion.py index 4c0b7d32134..3993e09becb 100644 --- a/tests/Integration/DataManagementSystem/FIXME_dfc_dir_completion.py +++ b/tests/Integration/DataManagementSystem/FIXME_dfc_dir_completion.py @@ -10,7 +10,6 @@ class DirCompletion(cmd.Cmd): - fc = FileCatalogClient() dfc_fs = DFCFileSystem(fc) dc = DirectoryCompletion(dfc_fs) @@ -52,6 +51,5 @@ def complete_ls(self, text, line, begidx, endidx): if __name__ == "__main__": - cli = DirCompletion() cli.cmdloop() diff --git a/tests/Integration/DataManagementSystem/Test_Client_DFC.py b/tests/Integration/DataManagementSystem/Test_Client_DFC.py index 3c953d6793c..125d74b2714 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_DFC.py +++ b/tests/Integration/DataManagementSystem/Test_Client_DFC.py @@ -890,7 +890,6 @@ def test_directoryOperations(self): if __name__ == "__main__": - res = getProxyInfo() if not res["OK"]: sys.exit(1) diff --git a/tests/Integration/DataManagementSystem/Test_Client_FTS3.py b/tests/Integration/DataManagementSystem/Test_Client_FTS3.py index 26606a572ce..143041f795a 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_FTS3.py +++ b/tests/Integration/DataManagementSystem/Test_Client_FTS3.py @@ -48,7 +48,6 @@ def generateOperation(self, opType, nbFiles, dests, sources=None): return op def test_01_operation(self): - op = self.generateOperation("Transfer", 3, ["Target1", "Target2"], sources=["Source1", "Source2"]) self.assertTrue(not op.isTotallyProcessed()) @@ -448,7 +447,6 @@ def test_05_cancelNotFoundJob(self): self.assertTrue(op.ftsFiles[3].status == "New") def _perf(self): - listOfIds = [] persistStart = time.time() diff --git a/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py b/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py index 8b70ce3a8c7..3fbfc8d6b5e 100644 --- a/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py +++ b/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py @@ -145,7 +145,6 @@ def test_seOperations(self): # create SE ret = self.db.addSE(seName, credDict) if isAdmin: - self.assertTrue(ret["OK"], "addSE failed when adding new SE: %s" % ret) seId = ret["Value"] @@ -996,7 +995,6 @@ def checkNonRecursiveDirectorySize(self, curDir): # If there are subdir, the recursive size of the subdir + the non recursive size of curdir # should be equal to the recursive size of curdir else: - # Get the logocal size of the subdirs ret = self.getAndCompareDirectorySize(subDirs, recursiveSum=True)["Value"]["Successful"] @@ -1488,7 +1486,6 @@ def _makeTestSuite(): if __name__ == "__main__": - managerTypes = list(MANAGER_TO_TEST) all_combinations = list(itertools.product(*MANAGER_TO_TEST.values())) numberOfManager = len(managerTypes) diff --git a/tests/Integration/Framework/NotRun_Test_ComponentInstallUninstall.py b/tests/Integration/Framework/NotRun_Test_ComponentInstallUninstall.py index 18c6baaeff2..c4cf9dbbb95 100644 --- a/tests/Integration/Framework/NotRun_Test_ComponentInstallUninstall.py +++ b/tests/Integration/Framework/NotRun_Test_ComponentInstallUninstall.py @@ -74,7 +74,6 @@ def tearDown(self): class ComponentInstallationChain(TestComponentInstallation): def testComponent(self): - service1Present = False service2Present = False @@ -168,7 +167,6 @@ def testComponent(self): ) def testDatabase(self): - gComponentInstaller.setMySQLPasswords(self.rootPwd, self.diracPwd) # Install database diff --git a/tests/Integration/Monitoring/Test_DataOperationSender.py b/tests/Integration/Monitoring/Test_DataOperationSender.py index d80c9b38fc7..87dfd75285c 100644 --- a/tests/Integration/Monitoring/Test_DataOperationSender.py +++ b/tests/Integration/Monitoring/Test_DataOperationSender.py @@ -124,6 +124,7 @@ }, ] + # fixture to have before the test methods @pytest.fixture def addToRegister(): diff --git a/tests/Integration/Monitoring/Test_MonitoringSystem.py b/tests/Integration/Monitoring/Test_MonitoringSystem.py index 57357e489cb..2eb84c30b4b 100644 --- a/tests/Integration/Monitoring/Test_MonitoringSystem.py +++ b/tests/Integration/Monitoring/Test_MonitoringSystem.py @@ -59,14 +59,12 @@ def putAndDelete(): def test_listReports(putAndDelete): - result = client.listReports("WMSHistory") assert result["OK"], result["Message"] assert result["Value"] == ["AverageNumberOfJobs", "NumberOfJobs", "NumberOfReschedules"] def test_listUniqueKeyValues(putAndDelete): - result = client.listUniqueKeyValues("WMSHistory") assert result["OK"], result["Message"] assert "Status" in result["Value"] @@ -90,7 +88,6 @@ def test_listUniqueKeyValues(putAndDelete): def test_generateDelayedPlot(putAndDelete): - params = ( "WMSHistory", "NumberOfJobs", @@ -119,7 +116,6 @@ def test_generateDelayedPlot(putAndDelete): def test_getReport(putAndDelete): - params = ( "WMSHistory", "NumberOfJobs", diff --git a/tests/Integration/ProductionSystem/Test_Client_TS_Prod.py b/tests/Integration/ProductionSystem/Test_Client_TS_Prod.py index dbf1124f5f0..72007c17189 100644 --- a/tests/Integration/ProductionSystem/Test_Client_TS_Prod.py +++ b/tests/Integration/ProductionSystem/Test_Client_TS_Prod.py @@ -47,7 +47,6 @@ def tearDown(self): class ProductionClientChain(TestClientProductionTestCase): def test_SeqProduction(self): - # Define the first step of the production prodStep1 = ProductionStep() prodStep1.Name = "Sim_prog" @@ -140,7 +139,6 @@ def test_SeqProduction(self): self.assertTrue(res["OK"]) def test_MergeProduction(self): - # Define the first step of the production prodStep1 = ProductionStep() prodStep1.Name = "Sim_prog" @@ -221,7 +219,6 @@ def test_MergeProduction(self): self.assertTrue(res["OK"]) def test_SplitProduction(self): - # Define the first step of the production prodStep1 = ProductionStep() prodStep1.Name = "Sim_prog" diff --git a/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py b/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py index b08eb5b794e..08b44c16990 100644 --- a/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py +++ b/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py @@ -34,6 +34,7 @@ from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient + ######################################################################## class FullChainTest: """ @@ -49,7 +50,6 @@ class FullChainTest: """ def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2): - files = self.files(owner, group) putAndRegister = Operation() @@ -148,7 +148,6 @@ def putRequest(self, userName, userDN, userGroup, sourceSE, targetSE1, targetSE2 # # test execution if __name__ == "__main__": - if len(sys.argv) != 5: gLogger.error("Usage:\n python %s userGroup SourceSE TargetSE1 TargetSE2\n") sys.exit(-1) diff --git a/tests/Integration/ResourceStatusSystem/Test_EmailActionAgent.py b/tests/Integration/ResourceStatusSystem/Test_EmailActionAgent.py index 535e3eb86d0..ae686b6080a 100644 --- a/tests/Integration/ResourceStatusSystem/Test_EmailActionAgent.py +++ b/tests/Integration/ResourceStatusSystem/Test_EmailActionAgent.py @@ -75,7 +75,6 @@ def test__emailActionAgent(): - rssClient = ResourceStatusClient() # clean up res = rssClient.delete("ResourceStatusCache") diff --git a/tests/Integration/ResourceStatusSystem/Test_FullChain.py b/tests/Integration/ResourceStatusSystem/Test_FullChain.py index 0198d169acd..dc762659b7d 100644 --- a/tests/Integration/ResourceStatusSystem/Test_FullChain.py +++ b/tests/Integration/ResourceStatusSystem/Test_FullChain.py @@ -133,7 +133,6 @@ def test_takeDecision_noDecisionParams(): ], ) def test_takeDecision_decisionParams(decisionParams, status): - # Arrange pdp.setup(decisionParams) diff --git a/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py b/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py index c07bbb48340..e85b427d66b 100644 --- a/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py +++ b/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py @@ -27,7 +27,6 @@ def fixtureResourceStatusClient(): def test_addAndRemove(rssClient: ResourceStatusClient): - # clean up rssClient.deleteStatusElement("Site", "Status", "TestSite1234") rssClient.deleteStatusElement("Site", "History", "TestSite1234") @@ -358,7 +357,6 @@ def test_addAndRemove(rssClient: ResourceStatusClient): def test_addIfNotThereStatusElement(rssClient: ResourceStatusClient): - # Clean up rssClient.deleteStatusElement("Resource", "Status", "TestName123456789") rssClient.deleteStatusElement("Resource", "History", "TestName123456789") diff --git a/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py b/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py index 67145f95d09..75e675ac4c4 100644 --- a/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py +++ b/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py @@ -29,7 +29,6 @@ def fixtureSiteStatus(): def test_addAndRemove_simpleCase(stClient): - # make sure that the test sites are not presented in the db rsClient = ResourceStatusClient() rsClient.deleteStatusElement("Site", "Status", testSite) diff --git a/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py b/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py index fb52c154368..71db0d42116 100644 --- a/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py +++ b/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py @@ -126,7 +126,6 @@ def pseudoCS(mqURI): class Test_MQProducers(unittest.TestCase): - reconnectWasCalled = False def setUp(self): @@ -212,7 +211,6 @@ def test_success(self, mock_getMQParamsFromCS): class Test_MQProducers_3(Test_MQProducers): @mock.patch("DIRAC.Resources.MessageQueue.MQCommunication.getMQParamsFromCS", side_effect=pseudoCS) def test_success(self, mock_getMQParamsFromCS): - conn1 = None # creating and closing for i in range(20): diff --git a/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py index 36964baf85f..73784fc3160 100644 --- a/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py +++ b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py @@ -10,7 +10,6 @@ class DummyTornadoHandler(TornadoService): - auth_true = ["all"] def export_true(self): diff --git a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py index 6eddde0042f..b2ced198093 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py +++ b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py @@ -60,7 +60,6 @@ @pytest.fixture(name="lfn") def lfnFixture(): - # Get VO res = getProxyInfo() assert res["OK"] @@ -282,7 +281,6 @@ def test_submitJob_parametricJob(): def test_WMSClient_rescheduleJob(): - # create the job job = helloWorldJob() job.setType("User") @@ -621,7 +619,6 @@ def test_JobStateUpdateAndJobMonitoringMultiple(lfn): def test_JobManagerClient_removeJob(): - # Arrange job = helloWorldJob() job.setType("User") diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py index 4a87565d9da..7945c020aeb 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py @@ -143,7 +143,6 @@ def fixturejobDB(): def test_insertAndRemoveJobIntoDB(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") assert res["OK"] is True, res["Message"] jobID = int(res["JobID"]) @@ -197,7 +196,6 @@ def test_insertAndRemoveJobIntoDB(jobDB): def test_rescheduleJob(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") assert res["OK"] is True, res["Message"] jobID = res["JobID"] @@ -218,13 +216,11 @@ def test_rescheduleJob(jobDB): def test_getCounters(jobDB): - res = jobDB.getCounters("Jobs", ["Status", "MinorStatus"], {}, "2007-04-22 00:00:00") assert res["OK"] is True, res["Message"] def test_heartBeatLogging(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") assert res["OK"] is True, res["Message"] jobID = res["JobID"] @@ -331,7 +327,6 @@ def test_setJobsMajorStatus(jobDB): def test_attributes(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner_1", "/DN/OF/owner", "ownerGroup", "someSetup") assert res["OK"] is True, res["Message"] jobID_1 = res["JobID"] diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobLoggingDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobLoggingDB.py index 39578d7efc0..de3445925cf 100755 --- a/tests/Integration/WorkloadManagementSystem/Test_JobLoggingDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobLoggingDB.py @@ -19,7 +19,6 @@ def fixtureJobLoggingDB(): def test_JobStatus(jobLoggingDB: JobLoggingDB): - result = jobLoggingDB.addLoggingRecord( 1, status="testing", diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobParameters_MySQLandES.py b/tests/Integration/WorkloadManagementSystem/Test_JobParameters_MySQLandES.py index 84c0974b85c..da44cd1c38f 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobParameters_MySQLandES.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobParameters_MySQLandES.py @@ -27,7 +27,6 @@ def createJob(): - job = helloWorldJob() res = Dirac().submitJob(job) diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py index 8f3703818fd..63d69153c43 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py @@ -32,7 +32,6 @@ class JobWrapperSubmissionCase(JobWrapperTestCase): """JobWrapperSubmissionCase represents a test suite for""" def test_CreateAndSubmit(self): - jobParams = { "JobID": "1", "JobType": "Merge", diff --git a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py index 1e63e9df11f..686342047e2 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py +++ b/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py @@ -21,7 +21,6 @@ def test_PilotsDB(): - pilots = PilotManagerClient() # This will allow you to run the test again if necessary diff --git a/tests/Integration/WorkloadManagementSystem/Test_PilotsLoggingClient.py b/tests/Integration/WorkloadManagementSystem/Test_PilotsLoggingClient.py index 2f93939c334..cb0171f9dea 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_PilotsLoggingClient.py +++ b/tests/Integration/WorkloadManagementSystem/Test_PilotsLoggingClient.py @@ -57,14 +57,12 @@ def test_PilotsLoggingAddGetDelete(self): self.assertEqual(resp["Value"], [], "PilotsLogging was not really deleted") def test_PilotsLoggingEmptyGetDelete(self): - resp = self.pilotsLoggingClient.getPilotsLogging("11111111-1111-1111-1111-111111111111") self.assertTrue(resp["OK"], "Failed to get PilotsLogging") resp = self.pilotsLoggingClient.deletePilotsLogging("11111111-1111-1111-1111-111111111111") self.assertTrue(resp["OK"], "Failed to delete PilotsLogging") def test_PilotsLoggingDeleteList(self): - test_sample1 = { "pilotUUID": "11111111-1111-1111-1111-111111111111", "timestamp": "timestamp1", diff --git a/tests/Integration/WorkloadManagementSystem/Test_WMSAdministratorClient.py b/tests/Integration/WorkloadManagementSystem/Test_WMSAdministratorClient.py index 37e6e5e9338..445997ba7f4 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_WMSAdministratorClient.py +++ b/tests/Integration/WorkloadManagementSystem/Test_WMSAdministratorClient.py @@ -9,7 +9,6 @@ def test_WMSAdministratorClient(): - wmsAdministrator = WMSAdministratorClient() sitesList = ["My.Site.org", "Your.Site.org"] diff --git a/tests/Integration/WorkloadManagementSystem/exe-script.py b/tests/Integration/WorkloadManagementSystem/exe-script.py index 7fa418e3f5d..0eba06df21e 100644 --- a/tests/Integration/WorkloadManagementSystem/exe-script.py +++ b/tests/Integration/WorkloadManagementSystem/exe-script.py @@ -6,5 +6,4 @@ # Main if __name__ == "__main__": - sys.exit(subprocess.call(shlex.split("echo 'Hello World'"))) diff --git a/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py b/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py index b19930b163f..562cebea43a 100644 --- a/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py +++ b/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py @@ -133,7 +133,6 @@ def setVerbosity(value): def listDirectory(n_queries): - global testDir start = time.time() @@ -144,7 +143,6 @@ def listDirectory(n_queries): startTotal = time.time() for i in range(n_queries): - start = time.time() result = fc.listDirectory(testDir) resultList.append(time.time() - start) @@ -168,7 +166,6 @@ def listDirectory(n_queries): def getBulkReplicas(n_queries): - global lfnListFile, verbosity lFile = open(lfnListFile) lfnList = [l.strip().replace("//", "/") for l in lFile.read().strip().split()] @@ -182,7 +179,6 @@ def getBulkReplicas(n_queries): startTotal = time.time() for i in range(n_queries): - start = time.time() result = fc.getReplicas(lfnList) resultList.append(time.time() - start) @@ -216,7 +212,6 @@ def getBulkReplicas(n_queries): def getDirectoryReplicas(n_queries): - global testDir, verbosity sCount = 0 @@ -226,7 +221,6 @@ def getDirectoryReplicas(n_queries): startTotal = time.time() for i in range(n_queries): - start = time.time() result = fc.getDirectoryReplicas(testDir) resultList.append(time.time() - start) @@ -266,7 +260,6 @@ def getDirectoryReplicas(n_queries): def finalize(task, result): - global resultTest, verbosity if verbosity >= 2: @@ -283,7 +276,6 @@ def doException(expt): def runTest(): - global nClients, nQueries, testType, resultTest, testDir, lfnListFile resultTest = [] @@ -319,7 +311,6 @@ def runTest(): def doStats(testArray): - array = list(testArray) # Delete min and max value first del array[array.index(max(array))] @@ -358,7 +349,6 @@ def doStats(testArray): def executeTest(nc, nf, queryDict, rateDict, queryDict_r, rateDict_r): - global nClients nClients = nc @@ -382,7 +372,6 @@ def executeTest(nc, nf, queryDict, rateDict, queryDict_r, rateDict_r): def runFullTest(): - global outputFile, nClients, testDir, lfnListFile, shortRange queryDict = {} diff --git a/tests/Performance/DFCPerformance/make_dfc_plots_lib.py b/tests/Performance/DFCPerformance/make_dfc_plots_lib.py index 4d1c0cc7f75..a17d2ededcd 100755 --- a/tests/Performance/DFCPerformance/make_dfc_plots_lib.py +++ b/tests/Performance/DFCPerformance/make_dfc_plots_lib.py @@ -78,7 +78,6 @@ def analyze_data(input_data, binSize=10): def make_plot( analyzed_data, base_filename="", plot_title="", disable_max=False, plot_filename=None, hist_filename=None ): - binStartTimes = analyzed_data["binStartTimes"] minQueryTimes = analyzed_data["minQueryTimes"] maxQueryTimes = analyzed_data["maxQueryTimes"] diff --git a/tests/Performance/DFCPerformance/mixedPerf.py b/tests/Performance/DFCPerformance/mixedPerf.py index 54b4e31d3a6..4a0c22fbb48 100755 --- a/tests/Performance/DFCPerformance/mixedPerf.py +++ b/tests/Performance/DFCPerformance/mixedPerf.py @@ -92,7 +92,6 @@ def doRead(depth): def doInsert(depth, maxFile): - dirPath = generatePath(depth, True) nbOfFiles = random.randint(1, maxFile) lfnDict = {} @@ -162,7 +161,6 @@ def weighted_choice(choices): while not done: - action = weighted_choice(proportions) if action == "r": diff --git a/tests/Performance/ESJobMonitoring/test_scripts/query.py b/tests/Performance/ESJobMonitoring/test_scripts/query.py index 481a7e8dddd..33610c6b09e 100644 --- a/tests/Performance/ESJobMonitoring/test_scripts/query.py +++ b/tests/Performance/ESJobMonitoring/test_scripts/query.py @@ -11,16 +11,13 @@ class Transaction: def __init__(self): - self.elasticJobParametersDB = ElasticJobParametersDB() self.custom_timers = {} def run(self): - start_time = time.time() for i in range(0, random.randint(1000, 3000)): - jobID = random.randint(1, 1000) self.elasticJobParametersDB.getJobParameters(jobID) @@ -30,7 +27,6 @@ def run(self): if __name__ == "__main__": - trans = Transaction() trans.run() print(trans.custom_timers) diff --git a/tests/Performance/ESJobMonitoring/test_scripts/update.py b/tests/Performance/ESJobMonitoring/test_scripts/update.py index 0f0b6c23721..ae65c41b5d8 100644 --- a/tests/Performance/ESJobMonitoring/test_scripts/update.py +++ b/tests/Performance/ESJobMonitoring/test_scripts/update.py @@ -16,16 +16,13 @@ def random_generator(size=6, chars=string.ascii_letters): class Transaction: def __init__(self): - self.elasticJobParametersDB = ElasticJobParametersDB() self.custom_timers = {} def run(self): - start_time = time.time() for i in range(0, random.randint(1000, 3000)): - key = random_generator() value = random_generator(size=12) self.elasticJobParametersDB.setJobParameter(2, key, value) @@ -36,7 +33,6 @@ def run(self): if __name__ == "__main__": - trans = Transaction() trans.run() print(trans.custom_timers) diff --git a/tests/Performance/MySQLJobMonitoring/test_scripts/query.py b/tests/Performance/MySQLJobMonitoring/test_scripts/query.py index e21c4f91dcf..8145e3cb4e2 100644 --- a/tests/Performance/MySQLJobMonitoring/test_scripts/query.py +++ b/tests/Performance/MySQLJobMonitoring/test_scripts/query.py @@ -11,16 +11,13 @@ class Transaction: def __init__(self): - self.JobDB = JobDB() self.custom_timers = {} def run(self): - start_time = time.time() for i in range(0, random.randint(1000, 3000)): - jobID = random.randint(1, 1000) self.JobDB.getJobParameters(jobID) @@ -30,7 +27,6 @@ def run(self): if __name__ == "__main__": - trans = Transaction() trans.run() print(trans.custom_timers) diff --git a/tests/Performance/MySQLJobMonitoring/test_scripts/update.py b/tests/Performance/MySQLJobMonitoring/test_scripts/update.py index ac362040eeb..4b2962f0d2d 100644 --- a/tests/Performance/MySQLJobMonitoring/test_scripts/update.py +++ b/tests/Performance/MySQLJobMonitoring/test_scripts/update.py @@ -16,16 +16,13 @@ def random_generator(size=6, chars=string.ascii_letters): class Transaction: def __init__(self): - self.JobDB = JobDB() self.custom_timers = {} def run(self): - start_time = time.time() for i in range(0, random.randint(1000, 3000)): - key = random_generator() value = random_generator(size=12) self.JobDB.setJobParameter(2, key, value) @@ -36,7 +33,6 @@ def run(self): if __name__ == "__main__": - trans = Transaction() trans.run() print(trans.custom_timers) diff --git a/tests/System/StorageTests/Test_Resources_StorageElement.py b/tests/System/StorageTests/Test_Resources_StorageElement.py index 394aafa402e..0d3482da3b2 100644 --- a/tests/System/StorageTests/Test_Resources_StorageElement.py +++ b/tests/System/StorageTests/Test_Resources_StorageElement.py @@ -99,7 +99,6 @@ def _mul(txt): @pytest.fixture def prepare_seObj_fixture(seName, protocolSection, prepare_local_testDir): - localWorkDir, destinationPath = prepare_local_testDir # When testing for a given plugin, this plugin might not be able to diff --git a/tests/Workflow/Integration/Test_UserJobs.py b/tests/Workflow/Integration/Test_UserJobs.py index 79dc48d6fec..ca8e33bde65 100644 --- a/tests/Workflow/Integration/Test_UserJobs.py +++ b/tests/Workflow/Integration/Test_UserJobs.py @@ -45,7 +45,6 @@ def setUp(self): class HelloWorldSuccess(UserJobTestCase): def test_execute(self): - j = Job() j.setName("helloWorld-test") @@ -65,7 +64,6 @@ class HelloWorldPlusSuccess(UserJobTestCase): """Adding quite a lot of calls from the API, for pure test purpose""" def test_execute(self): - job = Job() job._siteSet = {"DIRAC.someSite.ch"} @@ -102,7 +100,6 @@ def test_execute(self): self.assertTrue(res["OK"]) def test_execute_success(self): - job = Job() job._siteSet = {"DIRAC.someSite.ch"}