diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 6b20e226c85..dc7adf42182 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -91,26 +91,6 @@ jobs: env: REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} - pylint-py27: - name: Pylint for Python 2.7 in Pilot files - runs-on: ubuntu-latest - if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' - steps: - - uses: actions/checkout@v4 - - name: Fail-fast for outdated pipelines - run: .github/workflows/fail-fast.sh - - name: prepare environment - run: | - conda config --set add_pip_as_python_dependency false - conda create -c conda-forge -n test-env python=2.7 pylint=1.9.2 astroid=1.6.5 - - name: run pilot wrapper test - run: | - eval "$(conda shell.bash hook)" && conda activate test-env - pylint -j 0 -E \ - tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \ - src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \ - src/DIRAC/Resources/Computing/BatchSystems/*.py - diracx: name: DiracX Unit Tests runs-on: ubuntu-latest diff --git a/.github/workflows/pilotWrapper.yml b/.github/workflows/pilotWrapper.yml index 18159ed0810..823dd95e504 100644 --- a/.github/workflows/pilotWrapper.yml +++ b/.github/workflows/pilotWrapper.yml @@ -11,8 +11,6 @@ jobs: fail-fast: False matrix: python: - - 2.7.5 - - 2.7.13 - 3.6.8 - 3.11.4 pilot_branch: diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst index b26f3b4e9c3..6b641e524a1 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst @@ -40,7 +40,7 @@ where: - configure means adding dirac specific configuration files (which, at a minimum, should include the location of a DIRAC configuration service) -A pilot has to run on each and every computing resource type, provided that Python 2.6+ is on the WN. +A pilot has to run on each and every computing resource type, provided that Python 3.6+ is on the WN. The same pilot script can be used everywhere. .. image:: Pilots2.png diff --git a/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst b/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst index 49078f457d5..e5f96b7a126 100644 --- a/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst +++ b/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst @@ -11,7 +11,7 @@ Installing DIRAC client The DIRAC client installation procedure consists of few steps. You can do these steps as any user without the need to be root. -Python3 DIRAC installations rely on the environment provided by `DIRACOS2 `_. +DIRAC installations rely on the environment provided by `DIRACOS2 `_. So, you first install DIRACOS2 and only then install DIRAC in it:: $ curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-$(uname -m).sh diff --git a/setup.cfg b/setup.cfg index 9d999dfbe3a..c219f643d36 100644 --- a/setup.cfg +++ b/setup.cfg @@ -321,10 +321,9 @@ console_scripts = [pycodestyle] # Pep8 codes: # E203 - Whitespace before ':' (spaces should be present for list slices) -# E211 - Whitespace before '(' (black incorrectly formats exec when in Python 2 mode) # E266 - Too many leading '#' for block comment # E402 - module level import not at top of file (for scripts) # W503 - line break before binary operator (stupid, complains after or before...) -ignore = E203, E211, E266, E402, W503 +ignore = E203, E266, E402, W503 # TODO: This should be reduced back to 120 at some point max_line_length=130 diff --git a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py index 23a28944c59..eef7deb9786 100644 --- a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py +++ b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py @@ -41,8 +41,7 @@ def extractRequestFromFileId(fileId): if compressType == "Z": gLogger.info("Compressed request, uncompressing") try: - # Encoding is only required for Python 2 and can be removed when Python 2 support is no longer needed - stub = base64.urlsafe_b64decode(stub.encode()) + stub = base64.urlsafe_b64decode(stub) except Exception as e: gLogger.error("Oops! Plot request is not properly encoded!", str(e)) return S_ERROR(f"Oops! Plot request is not properly encoded!: {str(e)}") diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py index 388d112bdc1..a4e0b276b93 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Condor.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Condor.py @@ -3,9 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import json import re import tempfile diff --git a/src/DIRAC/Resources/Computing/BatchSystems/GE.py b/src/DIRAC/Resources/Computing/BatchSystems/GE.py index 4590ecffb75..e01b109c930 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/GE.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/GE.py @@ -14,9 +14,6 @@ SubmitOption = -l ct=6000 """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import shlex import subprocess diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Host.py b/src/DIRAC/Resources/Computing/BatchSystems/Host.py index cdd0a49396b..3b827232013 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Host.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Host.py @@ -8,10 +8,6 @@ with LocalComputingElement or SSHComputingElement objects """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import glob import shutil diff --git a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py index 4d91e92ffe8..6850a0c7dbf 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/LSF.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/LSF.py @@ -9,9 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division import re import subprocess import shlex diff --git a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py index dfb0fb5d462..24e73bdf1b9 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/OAR.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/OAR.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import shlex import os diff --git a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py index b66ce035069..86484f58343 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/SLURM.py @@ -3,10 +3,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import re import subprocess diff --git a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py index 22093565682..6eb44670bde 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/Torque.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/Torque.py @@ -9,10 +9,6 @@ LocalComputingElement and SSHComputingElement classes """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import subprocess import shlex import os diff --git a/src/DIRAC/Resources/Computing/SSHComputingElement.py b/src/DIRAC/Resources/Computing/SSHComputingElement.py index dc353bbdcac..53ea1117eb0 100644 --- a/src/DIRAC/Resources/Computing/SSHComputingElement.py +++ b/src/DIRAC/Resources/Computing/SSHComputingElement.py @@ -487,9 +487,11 @@ def __executeHostCommand(self, command, options, ssh=None, host=None): options = json.dumps(options) options = quote(options) - cmd = ( - "bash --login -c 'python3 %s/execute_batch %s || python %s/execute_batch %s || python2 %s/execute_batch %s'" - % (self.sharedArea, options, self.sharedArea, options, self.sharedArea, options) + cmd = "bash --login -c 'python3 {}/execute_batch {} || python {}/execute_batch {}'".format( + self.sharedArea, + options, + self.sharedArea, + options, ) self.log.verbose(f"CE submission command: {cmd}") diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py index a60c7839c3b..50e1eef3d89 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -32,8 +32,6 @@ py='python' elif command -v python3 &> /dev/null; then py='python3' -elif command -v python2 &> /dev/null; then - py='python2' fi /usr/bin/env $py << EOF @@ -63,14 +61,8 @@ import shlex from uuid import uuid1 -try: - # For Python 3.0 and later - from urllib.request import urlopen, HTTPError, URLError - from urllib.parse import urlencode -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, HTTPError, URLError - from urllib import urlencode +from urllib.request import urlopen, HTTPError, URLError +from urllib.parse import urlencode try: from cStringIO import StringIO @@ -93,10 +85,7 @@ def formatTime(self, record, datefmt=None): # formatter = logging.Formatter(fmt='%%(asctime)s UTC %%(levelname)-8s %%(message)s', datefmt='%%Y-%%m-%%d %%H:%%M:%%S') formatter = MicrosecondFormatter('%%(asctime)s %%(levelname)-8s [%%(name)s] %%(message)s') logging.Formatter.converter = time.gmtime -try: - screen_handler = logging.StreamHandler(stream=sys.stdout) -except TypeError: # python2.6 - screen_handler = logging.StreamHandler(strm=sys.stdout) +screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) # add a string buffer handler @@ -276,28 +265,15 @@ def pilotWrapperScript( # Getting the json, tar, and checksum file try: - # urllib is different between python 2 and 3 - if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen - from urllib2 import URLError as url_library_URLError - else: - from urllib.request import urlopen as url_library_urlopen - from urllib.error import URLError as url_library_URLError + from urllib.request import urlopen + from urllib.error import URLError for fileName in ['checksums.sha512', 'pilot.json', 'pilot.tar']: - # needs to distinguish whether urlopen method contains the 'context' param - # in theory, it should be available from python 2.7.9 - # in practice, some prior versions may be composed of recent urllib version containing the param - if 'context' in url_library_urlopen.__code__.co_varnames: - import ssl - context = ssl._create_unverified_context() - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10, - context=context) - - else: - remoteFile = url_library_urlopen(os.path.join(loc, fileName), - timeout=10) + import ssl + context = ssl._create_unverified_context() + remoteFile = urlopen(os.path.join(loc, fileName), + timeout=10, + context=context) localFile = open(fileName, 'wb') localFile.write(remoteFile.read()) @@ -320,7 +296,7 @@ def pilotWrapperScript( raise # if we get here we break out of the loop of locations break - except (url_library_URLError, Exception) as e: + except (URLError, Exception) as e: print('%%s unreacheable (this is normal!)' %% loc, file=sys.stderr) logger.error('%%s unreacheable (this is normal!)' %% loc) logger.exception(e) diff --git a/tests/CI/run_pilot.sh b/tests/CI/run_pilot.sh index f7827b8503f..bcc415309ae 100755 --- a/tests/CI/run_pilot.sh +++ b/tests/CI/run_pilot.sh @@ -31,15 +31,13 @@ cp /ca/certs/pilot_proxy /tmp/x509up_u$UID eval "${PILOT_DOWNLOAD_COMMAND}" -echo "${PILOT_JSON}" > pilot.json -jq < pilot.json +echo "${PILOT_JSON}" >pilot.json +jq /dev/null; then +if command -v python &>/dev/null; then py='python' -elif command -v python3 &> /dev/null; then +elif command -v python3 &>/dev/null; then py='python3' -elif command -v python2 &> /dev/null; then - py='python2' fi # shellcheck disable=SC2086 diff --git a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py index 7c93b06d458..81dc37d1375 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py @@ -6,8 +6,6 @@ # - starts it # # It should be executed for different versions of python, e.g.: -# - 2.7.x (x < 9) -# - 2.7.x (x >= 9) # - 3.6.x # - 3.11.x # @@ -25,20 +23,12 @@ # 1) gets the (DIRAC-free) PilotWrapper.py -# urllib is different between python 2 and 3 -if sys.version_info < (3,): - from urllib2 import urlopen as url_library_urlopen # pylint: disable=import-error -else: - from urllib.request import urlopen as url_library_urlopen # pylint: disable=import-error,no-name-in-module +from urllib.request import urlopen # pylint: disable=import-error,no-name-in-module +import ssl # pylint: disable=import-error -if sys.version_info >= (2, 7, 9): - import ssl # pylint: disable=import-error - - context = ssl._create_unverified_context() - rf = url_library_urlopen(sys.argv[1], context=context) -else: - rf = url_library_urlopen(sys.argv[1]) +context = ssl._create_unverified_context() +rf = urlopen(sys.argv[1], context=context) locc = sys.argv[2] with open("PilotWrapper.py", "wb") as pj: