diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index e0f9221b..629d740e 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,7 +24,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -55,7 +54,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 @@ -82,7 +80,6 @@ jobs: fail-fast: false matrix: python: - - 2.7.18 - 3.6.15 - 3.9.17 diff --git a/.pylintrc b/.pylintrc index 2bae85fd..7c158f91 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,3 @@ dummy-variables=_ disable= invalid-name, line-too-long, # would be nice to remove this one - consider-using-f-string, # python2/3 support - unspecified-encoding, # python2/3 support - super-with-arguments, # python2/3 support - redefined-builtin, # python2/3 support \ No newline at end of file diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 9c434c97..b3f0866c 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -19,36 +19,19 @@ But, as said, all the actions are actually configurable. """ -from __future__ import absolute_import, division, print_function - import os import sys import time +from io import StringIO -############################ -# python 2 -> 3 "hacks" - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from .pilotTools import ( + Logger, + PilotParams, + RemoteLogger, + getCommand, + pythonPathCheck, +) -try: - from Pilot.pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) -except ImportError: - from pilotTools import ( - Logger, - PilotParams, - RemoteLogger, - getCommand, - pythonPathCheck, - ) ############################ if __name__ == "__main__": diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 4815f44b..7411f4a5 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -17,8 +17,6 @@ def __init__(self, pilotParams): execution. """ -from __future__ import absolute_import, division, print_function - import filecmp import os import platform @@ -28,39 +26,18 @@ def __init__(self, pilotParams): import sys import time import traceback -import subprocess from collections import Counter +from http.client import HTTPSConnection +from shlex import quote -############################ -# python 2 -> 3 "hacks" -try: - # For Python 3.0 and later - from http.client import HTTPSConnection -except ImportError: - # Fall back to Python 2 - from httplib import HTTPSConnection - -try: - from shlex import quote -except ImportError: - from pipes import quote - -try: - from Pilot.pilotTools import ( - CommandBase, - getSubmitterInfo, - retrieveUrlTimeout, - safe_listdir, - sendMessage, - ) -except ImportError: - from pilotTools import ( +from .pilotTools import ( CommandBase, getSubmitterInfo, retrieveUrlTimeout, safe_listdir, sendMessage, ) + ############################ @@ -283,7 +260,7 @@ def _getPreinstalledEnvScript(self): self.pp.installEnv["DIRAC_RC_PATH"] = preinstalledEnvScript def _localInstallDIRAC(self): - """Install python3 version of DIRAC client""" + """Install DIRAC client""" self.log.info("Installing DIRAC locally") @@ -296,10 +273,7 @@ def _localInstallDIRAC(self): # 1. Get the DIRACOS installer name # curl -O -L https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh - try: - machine = os.uname().machine # py3 - except AttributeError: - machine = os.uname()[4] # py2 + machine = os.uname().machine installerName = "DIRACOS-Linux-%s.sh" % machine diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8afe0f62..7d9530ad 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1,9 +1,8 @@ """A set of common tools to be used in pilot commands""" -from __future__ import absolute_import, division, print_function - import fcntl import getopt +import importlib.util import json import os import re @@ -16,81 +15,23 @@ import warnings from datetime import datetime from functools import partial, wraps -from threading import RLock - -############################ -# python 2 -> 3 "hacks" -try: - from urllib.error import HTTPError, URLError - from urllib.parse import urlencode - from urllib.request import urlopen -except ImportError: - from urllib import urlencode - - from urllib2 import HTTPError, URLError, urlopen - -try: - import importlib.util - from importlib import import_module - - def load_module_from_path(module_name, path_to_module): - spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member - module = importlib.util.module_from_spec(spec) # pylint: disable=no-member - spec.loader.exec_module(module) - return module - -except ImportError: +from importlib import import_module +from io import StringIO +from threading import RLock, Timer +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import urlopen - def import_module(module): - import imp +from .proxyTools import getVO - impData = imp.find_module(module) - return imp.load_module(module, *impData) +# Utilities functions - def load_module_from_path(module_name, path_to_module): - import imp - fp, pathname, description = imp.find_module(module_name, [path_to_module]) - try: - return imp.load_module(module_name, fp, pathname, description) - finally: - if fp: - fp.close() - - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - basestring # pylint: disable=used-before-assignment -except NameError: - basestring = str - -try: - from Pilot.proxyTools import getVO -except ImportError: - from proxyTools import getVO - -try: - FileNotFoundError # pylint: disable=used-before-assignment - # because of https://github.com/PyCQA/pylint/issues/6748 -except NameError: - FileNotFoundError = OSError - -try: - IsADirectoryError # pylint: disable=used-before-assignment -except NameError: - IsADirectoryError = IOError - -# Timer 2.7 and < 3.3 versions issue where Timer is a function -if sys.version_info.major == 2 or sys.version_info.major == 3 and sys.version_info.minor < 3: - from threading import _Timer as Timer # pylint: disable=no-name-in-module -else: - from threading import Timer - -# Utilities functions +def load_module_from_path(module_name, path_to_module): + spec = importlib.util.spec_from_file_location(module_name, path_to_module) # pylint: disable=no-member + module = importlib.util.module_from_spec(spec) # pylint: disable=no-member + spec.loader.exec_module(module) + return module def parseVersion(releaseVersion): @@ -399,7 +340,7 @@ def loadModule(self, modName, hideExceptions=False): def __recurseImport(self, modName, parentModule=None, hideExceptions=False): """Internal function to load modules""" - if isinstance(modName, basestring): + if isinstance(modName, str): modName = modName.split(".") try: if parentModule: @@ -713,11 +654,7 @@ def sendMessage(url, pilotUUID, wnVO, method, rawMessage): context.load_cert_chain(os.path.join(cert, "hostcert.pem"), os.path.join(cert, "hostkey.pem")) raw_data = {"method": method, "args": message, "extraCredentials": '"hosts"'} - if sys.version_info.major == 3: - data = urlencode(raw_data).encode("utf-8") # encode to bytes ! for python3 - else: - # Python2 - data = urlencode(raw_data) + data = urlencode(raw_data).encode("utf-8") # encode to bytes res = urlopen(url, data, context=context) res.close() @@ -787,17 +724,7 @@ def executeAndGetOutput(self, cmd, environDict=None): if not outChunk: continue dataWasRead = True - if sys.version_info.major == 2: - # Ensure outChunk is unicode in Python 2 - if isinstance(outChunk, str): - outChunk = outChunk.decode("utf-8") - # Strip unicode replacement characters - # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace(u"\ufffd", "")) - # Avoid potential str() issues in Py2 - outChunk = unicode(outChunk) # pylint: disable=undefined-variable - else: - outChunk = str(outChunk.replace("\ufffd", "")) # Python 3: Ensure it's a string + outChunk = str(outChunk.replace("\ufffd", "")) # Ensure it's a string if stream == _p.stderr: sys.stderr.write(outChunk) @@ -1455,7 +1382,7 @@ def __initJSON(self): # Commands first # FIXME: pilotSynchronizer() should publish these as comma-separated lists. We are ready for that. try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"][self.gridCEType].split(",") @@ -1466,7 +1393,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["Commands"]["Defaults"].split(",") @@ -1477,7 +1404,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], basestring): + if isinstance(self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["Commands"][self.gridCEType].split(",") @@ -1488,7 +1415,7 @@ def __initJSON(self): ] except KeyError: try: - if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], basestring): + if isinstance(self.pilotJSON["Defaults"]["Commands"]["Defaults"], str): self.commands = [ str(pv).strip() for pv in self.pilotJSON["Defaults"]["Commands"]["Defaults"].split(",") ] @@ -1504,7 +1431,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that. try: if isinstance( - self.pilotJSON["Setups"][self.setup]["CommandExtensions"], basestring + self.pilotJSON["Setups"][self.setup]["CommandExtensions"], str ): # In the specific setup? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["CommandExtensions"].split(",") @@ -1516,7 +1443,7 @@ def __initJSON(self): except KeyError: try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], basestring + self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"], str ): # Or in the defaults section? self.commandExtensions = [ str(pv).strip() for pv in self.pilotJSON["Setups"]["Defaults"]["CommandExtensions"].split(",") @@ -1533,7 +1460,7 @@ def __initJSON(self): # pilotSynchronizer() can publish this as a comma separated list. We are ready for that try: if isinstance( - self.pilotJSON["ConfigurationServers"], basestring + self.pilotJSON["ConfigurationServers"], str ): # Generic, there may also be setup-specific ones self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"].split(",")] @@ -1544,7 +1471,7 @@ def __initJSON(self): pass try: # now trying to see if there is setup-specific ones if isinstance( - self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], basestring + self.pilotJSON["Setups"][self.setup]["ConfigurationServer"], str ): # In the specific setup? self.configServer = ",".join( [str(pv).strip() for pv in self.pilotJSON["Setups"][self.setup]["ConfigurationServer"].split(",")] @@ -1556,7 +1483,7 @@ def __initJSON(self): except KeyError: # and if it doesn't exist try: if isinstance( - self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], basestring + self.pilotJSON["Setups"]["Defaults"]["ConfigurationServer"], str ): # Is there one in the defaults section? self.configServer = ",".join( [ diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..8792a34a 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -1,7 +1,5 @@ """few functions for dealing with proxies""" -from __future__ import absolute_import, division, print_function - import re from base64 import b16decode from subprocess import PIPE, Popen diff --git a/Pilot/tests/Test_Pilot.py b/Pilot/tests/Test_Pilot.py index 8a1b75a1..bf0f54c1 100644 --- a/Pilot/tests/Test_Pilot.py +++ b/Pilot/tests/Test_Pilot.py @@ -1,7 +1,5 @@ """Test class for Pilot""" -from __future__ import absolute_import, division, print_function - import json import os import shutil @@ -12,8 +10,8 @@ # imports import unittest -from Pilot.pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes -from Pilot.pilotTools import PilotParams +from ..pilotCommands import CheckWorkerNode, ConfigureSite, NagiosProbes +from ..pilotTools import PilotParams class PilotTestCase(unittest.TestCase): diff --git a/Pilot/tests/Test_proxyTools.py b/Pilot/tests/Test_proxyTools.py index 7a8688cb..dc6c3a95 100644 --- a/Pilot/tests/Test_proxyTools.py +++ b/Pilot/tests/Test_proxyTools.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, division, print_function import os import shlex @@ -6,19 +5,9 @@ import subprocess import sys import unittest +from unittest.mock import patch -############################ -# python 2 -> 3 "hacks" -try: - from Pilot.proxyTools import getVO, parseASN1 -except ImportError: - from proxyTools import getVO, parseASN1 - -try: - from unittest.mock import patch -except ImportError: - from mock import patch - +from ..proxyTools import getVO, parseASN1 class TestProxyTools(unittest.TestCase): def test_getVO(self): @@ -92,46 +81,7 @@ def __createFakeProxy(self, proxyFile): """ Create a fake proxy locally. """ - basedir = os.path.dirname(__file__) - shutil.copy(basedir + "/certs/user/userkey.pem", basedir + "/certs/user/userkey400.pem") - os.chmod(basedir + "/certs/user/userkey400.pem", 0o400) - ret = self.createFakeProxy( - basedir + "/certs/user/usercert.pem", - basedir + "/certs/user/userkey400.pem", - "fakeserver.cern.ch:15000", - "fakevo", - basedir + "/certs//host/hostcert.pem", - basedir + "/certs/host/hostkey.pem", - basedir + "/certs/ca", - proxyFile, - ) - os.remove(basedir + "/certs/user/userkey400.pem") - return ret - - def createFakeProxy(self, usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile): - """ - voms-proxy-fake --cert usercert.pem - --key userkey.pem - -rfc - -fqan "/fakevo/Role=user/Capability=NULL" - -uri fakeserver.cern.ch:15000 - -voms fakevo - -hostcert hostcert.pem - -hostkey hostkey.pem - -certdir ca - """ - opt = ( - '--cert %s --key %s -rfc -fqan "/fakevo/Role=user/Capability=NULL" -uri %s -voms %s -hostcert %s' - " -hostkey %s -certdir %s -out %s" - % (usercert, userkey, serverURI, vo, hostcert, hostkey, CACertDir, proxyfile) - ) - proc = subprocess.Popen( - shlex.split("voms-proxy-fake " + opt), - bufsize=1, - stdout=sys.stdout, - stderr=sys.stderr, - universal_newlines=True, - ) - proc.communicate() - return proc.returncode + shutil.copy(basedir + "/certs/voms/proxy.pem", proxyFile) + return 0 + diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 1fc448ae..aec1b191 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -1,26 +1,14 @@ #!/usr/bin/env python -from __future__ import absolute_import, division, print_function - import json import os import random import string -import sys import tempfile - -try: - from Pilot.pilotTools import CommandBase, Logger, PilotParams -except ImportError: - from pilotTools import CommandBase, Logger, PilotParams - import unittest +from unittest.mock import patch -try: - from unittest.mock import patch -except ImportError: - from mock import patch - +from ..pilotTools import CommandBase, Logger, PilotParams class TestPilotParams(unittest.TestCase): @patch("sys.argv") @@ -146,16 +134,10 @@ def test_executeAndGetOutput(self, popenMock, argvmock): for size in [1000, 1024, 1025, 2005]: random_str = "".join(random.choice(string.ascii_letters + "\n") for i in range(size)) - if sys.version_info.major == 3: - random_bytes = random_str.encode("UTF-8") - self.stdout_mock.write(random_bytes) - else: - self.stdout_mock.write(random_str) + random_bytes = random_str.encode("UTF-8") + self.stdout_mock.write(random_bytes) self.stdout_mock.seek(0) - if sys.version_info.major == 3: - self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) - else: - self.stderr_mock.write("Errare humanum est!") + self.stderr_mock.write("Errare humanum est!".encode("UTF-8")) self.stderr_mock.seek(0) pp = PilotParams() diff --git a/Untitled b/Untitled new file mode 100644 index 00000000..10dd1dde --- /dev/null +++ b/Untitled @@ -0,0 +1,11 @@ + UPDATE sb_SandBoxes AS s + JOIN ( + SELECT + MAX(OwnerId) AS badId, + MIN(OwnerId) AS goodId + FROM sb_Owners + GROUP BY Owner, OwnerGroup, VO + HAVING COUNT(*) > 1 + ) AS xxx + ON s.OwnerId = xxx.badId + SET s.OwnerId = xxx.goodId; diff --git a/environment.yml b/environment.yml index 41e0a564..72e2765e 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - requests # testing and development - pycodestyle - - caniusepython3 - coverage - mock - pylint