Skip to content

Commit 75e9ec8

Browse files
wdpyperewdpypere
committed
make python3 only and some cleanup
1 parent e772ce7 commit 75e9ec8

25 files changed

+61
-92
lines changed

‎Jenkinsfile

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ node {
99
sh 'git clean -fxd'
1010
}
1111
stage('test') {
12-
sh 'python2.7 -V'
1312
sh 'pip3 install --ignore-installed --prefix $PWD/.vsc-tox tox'
1413
sh 'export PATH=$PWD/.vsc-tox/bin:$PATH && export PYTHONPATH=$PWD/.vsc-tox/lib/python$(python3 -c "import sys; print(\\"%s.%s\\" % sys.version_info[:2])")/site-packages:$PYTHONPATH && tox -v -c tox.ini'
1514
sh 'rm -r $PWD/.vsc-tox'

‎README.md

+1-8
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
# vsc-utils
22

3-
### Build Status
4-
- Python 2.6 : [![Build Status](https://jenkins1.ugent.be/job/vsc-utils-python26/badge/icon)](https://jenkins1.ugent.be/job/vsc-utils-python26/)
5-
- Python 2.7 : [![Build Status](https://jenkins1.ugent.be/job/vsc-utils-python27/badge/icon)](https://jenkins1.ugent.be/job/vsc-utils-python27/)
6-
73
# Description
84
Common tools used within our organization.
95
These tools live in the vsc.utils namespace, toghether with the tools from
@@ -12,8 +8,5 @@ vsc-base.
128
They have been split of from vsc-base because the tools here need aditional dependencies
139
to work correctly whereas the tools in vsc-base do not.
1410

15-
Originally created by the HPC team of Ghent University (http://ugent.be/hpc).
16-
17-
# Documentation
18-
https://jenkins1.ugent.be/job/vsc-utils-python26/Documentation/
11+
Originally created by the HPC team of Ghent University (https://ugent.be/hpc).
1912

‎lib/vsc/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2015-2021 Ghent University
2+
# Copyright 2015-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2015-2021 Ghent University
2+
# Copyright 2015-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/availability.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/cache.py

+17-20
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -32,10 +32,9 @@
3232
import jsonpickle
3333
import os
3434
import time
35+
import pickle
3536

3637
from vsc.utils import fancylogger
37-
from vsc.utils.py2vs3 import pickle, FileNotFoundErrorExc
38-
3938

4039
class FileCache(object):
4140
"""File cache with a timestamp safety.
@@ -106,7 +105,7 @@ def __init__(self, filename, retain_old=True, raise_unpickable=False):
106105
finally:
107106
g.close()
108107

109-
except (OSError, IOError, ValueError, FileNotFoundErrorExc) as err:
108+
except (OSError, IOError, ValueError, FileNotFoundError) as err:
110109
self.log.warning("Could not access the file cache at %s [%s]", self.filename, err)
111110
self.shelf = {}
112111
self.log.info("Cache in %s starts with an empty shelf", (self.filename,))
@@ -158,19 +157,17 @@ def close(self):
158157
dirname = os.path.dirname(self.filename)
159158
if not os.path.exists(dirname):
160159
os.makedirs(dirname)
161-
f = open(self.filename, 'wb')
162-
if not f:
163-
self.log.error('cannot open the file cache at %s for writing', self.filename)
164-
else:
165-
if self.retain_old:
166-
self.shelf.update(self.new_shelf)
167-
self.new_shelf = self.shelf
168-
169-
g = gzip.GzipFile(mode='wb', fileobj=f)
170-
pickled = jsonpickle.encode(self.new_shelf)
171-
# .encode() is required in Python 3, since we need to pass a bytestring
172-
g.write(pickled.encode())
173-
g.close()
174-
f.close()
175-
176-
self.log.info('closing the file cache at %s', self.filename)
160+
with open(self.filename, 'wb') as fih:
161+
if not fih:
162+
self.log.error('cannot open the file cache at %s for writing', self.filename)
163+
else:
164+
if self.retain_old:
165+
self.shelf.update(self.new_shelf)
166+
self.new_shelf = self.shelf
167+
168+
with gzip.GzipFile(mode='wb', fileobj=fih) as zipf:
169+
pickled = jsonpickle.encode(self.new_shelf)
170+
# .encode() is required in Python 3, since we need to pass a bytestring
171+
zipf.write(pickled.encode())
172+
173+
self.log.info('closing the file cache at %s', self.filename)

‎lib/vsc/utils/fs_store.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/lock.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/nagios.py

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
# -*- encoding: utf-8 -*-
33
#
4-
# Copyright 2012-2021 Ghent University
4+
# Copyright 2012-2023 Ghent University
55
#
66
# This file is part of vsc-utils,
77
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -40,7 +40,6 @@
4040
@author: Andy Georges (Ghent University)
4141
@author: Luis Fernando Muñoz Mejías (Ghent University)
4242
"""
43-
from __future__ import print_function
4443

4544
import operator
4645
import os
@@ -52,7 +51,6 @@
5251

5352
from vsc.utils.cache import FileCache
5453
from vsc.utils.fancylogger import getLogger
55-
from vsc.utils.py2vs3 import is_string, FileNotFoundErrorExc
5654

5755
log = getLogger(__name__)
5856

@@ -155,7 +153,7 @@ def __init__(self, nrange):
155153
"""
156154
self.log = getLogger(self.__class__.__name__, fname=False)
157155

158-
if not is_string(nrange):
156+
if not isinstance(nrange, str):
159157
newnrange = str(nrange)
160158
self.log.debug("nrange %s of type %s, converting to string (%s)", str(nrange), type(nrange), newnrange)
161159
try:
@@ -316,7 +314,7 @@ def cache(self, nagios_exit, nagios_message):
316314
else:
317315
self.log.warn("Not running as root: Cannot chown the nagios check file %s to %s",
318316
self.filename, self.nagios_username)
319-
except (OSError, FileNotFoundErrorExc):
317+
except (OSError, FileNotFoundError):
320318
self.log.raiseException("Cannot chown the nagios check file %s to the nagios user" % (self.filename))
321319

322320
return True

‎lib/vsc/utils/pickle_files.py

+7-8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,10 +38,7 @@
3838
import logging
3939
import os
4040
import stat
41-
42-
43-
from vsc.utils.py2vs3 import pickle, FileNotFoundErrorExc
44-
41+
import pickle
4542

4643
class TimestampPickle(object):
4744
"""Stores a timestamp in some format in a file."""
@@ -60,8 +57,9 @@ def read(self):
6057
"""
6158

6259
try:
63-
timestamp = pickle.load(open(self.filename, "rb"))
64-
except (IOError, OSError, FileNotFoundErrorExc):
60+
with open (self.filename, "rb") as fih:
61+
timestamp = pickle.load(fih)
62+
except (IOError, OSError, FileNotFoundError):
6563
logging.exception("Failed to load timestamp pickle from filename %s.", self.filename)
6664
return None
6765

@@ -78,7 +76,8 @@ def write(self, timestamp):
7876
"""
7977

8078
try:
81-
pickle.dump(timestamp, open(self.filename, "wb"))
79+
with open(self.filename, "wb") as fih:
80+
pickle.dump(timestamp, fih)
8281
except Exception:
8382
logging.exception("Failed to dump timestamp %s to pickle in filename %s", timestamp, self.filename)
8483
raise

‎lib/vsc/utils/rest_oauth.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -28,9 +28,8 @@
2828
was registered with the OAuth system of the web application.
2929
"""
3030
import jsonpickle
31-
32-
from vsc.utils.py2vs3 import Request, urlencode
33-
31+
from urllib.parse import urlencode
32+
from urllib.request import Request
3433

3534
def request_access_token(opener, path, client_id, client_secret):
3635
"""

‎lib/vsc/utils/script_tools.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎lib/vsc/utils/timestamp.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# -*- coding: latin-1 -*-
22
#
3-
# Copyright 2009-2021 Ghent University
3+
# Copyright 2009-2023 Ghent University
44
#
55
# This file is part of vsc-utils,
66
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -36,7 +36,6 @@
3636

3737
from vsc.utils.cache import FileCache
3838
from vsc.utils.dateandtime import utc
39-
from vsc.utils.py2vs3 import is_string
4039

4140
LDAP_DATETIME_TIMEFORMAT = "%Y%m%d%H%M%SZ"
4241

@@ -66,7 +65,7 @@ def convert_to_datetime(timestamp=None):
6665
if isinstance(timestamp, datetime.datetime):
6766
if timestamp.tzinfo is None:
6867
timestamp = timestamp.replace(tzinfo=utc)
69-
elif is_string(timestamp):
68+
elif isinstance(timestamp, str):
7069
if len(timestamp) == 10:
7170
# Unix timestamp
7271
timestamp = datetime.datetime.fromtimestamp(int(timestamp), utc)

‎lib/vsc/utils/timestamp_pid_lockfile.py

+9-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -38,7 +38,6 @@
3838
import time
3939

4040
from lockfile.linklockfile import LockBase, LockFailed, NotLocked, NotMyLock
41-
from vsc.utils.py2vs3 import FileNotFoundErrorExc, FileExistsErrorExc
4241

4342
class LockFileReadError(Exception):
4443
'''Exception raised when we cannot get the expected information from the lock file.'''
@@ -83,7 +82,7 @@ def acquire(self, timeout=None):
8382
try:
8483
_write_pid_timestamp_file(self.path)
8584
logging.info('Obtained lock on timestamped pid lockfile %s', self.path)
86-
except (OSError, FileNotFoundErrorExc, FileExistsErrorExc) as err:
85+
except (OSError, FileNotFoundError, FileExistsError) as err:
8786
doraise = True
8887
if err.errno == errno.EEXIST:
8988
## Check if the timestamp is older than the threshold
@@ -125,20 +124,18 @@ def _read_pid_timestamp_file(path):
125124
Returns (pid :: Int, timestamp :: Int).
126125
'''
127126
try:
128-
pidfp = open(path, 'r')
127+
with open(path, 'r') as pidfp:
128+
pidLine = pidfp.readline().strip()
129+
timestampLine = pidfp.readline().strip()
130+
pid = int(pidLine)
131+
timestamp = int(timestampLine)
132+
return (pid, timestamp)
133+
129134
except IOError as err:
130135
if err.errno == errno.ENOENT:
131136
return None
132137
else:
133138
raise LockFileReadError('Cannot get the information from the pid file.')
134-
135-
pidLine = pidfp.readline().strip()
136-
timestampLine = pidfp.readline().strip()
137-
138-
try:
139-
pid = int(pidLine)
140-
timestamp = int(timestampLine)
141-
return (pid, timestamp)
142139
except ValueError:
143140
raise LockFileReadError("Contents of pid file %s invalid" % (path))
144141

‎setup.py

+1-14
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@
3030
@author: Stijn De Weirdt (Ghent University)
3131
@author: Andy Georges (Ghent University)
3232
"""
33-
import sys
34-
3533
import vsc.install.shared_setup as shared_setup
3634
from vsc.install.shared_setup import ag, sdw
3735

@@ -41,20 +39,9 @@
4139
'netifaces',
4240
'jsonpickle',
4341
]
44-
if sys.version_info < (3, 0):
45-
# jsonpickle pulls in too new and wrong deps on CentOS 7
46-
install_requires.extend([
47-
'jsonpickle < 1.4.0'
48-
])
49-
50-
else:
51-
install_requires.extend([
52-
'jsonpickle'
53-
])
54-
5542

5643
PACKAGE = {
57-
'version': '2.1.10',
44+
'version': '2.2.0',
5845
'author': [ag, sdw],
5946
'maintainer': [ag, sdw],
6047
'excluded_pkgs_rpm': ['vsc', 'vsc.utils'], # vsc is default, vsc.utils is provided by vsc-base

‎test/00-import.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2016-2021 Ghent University
2+
# Copyright 2016-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎test/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2016-2021 Ghent University
2+
# Copyright 2016-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎test/cache.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

‎test/nagios.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#
2-
# Copyright 2012-2021 Ghent University
2+
# Copyright 2012-2023 Ghent University
33
#
44
# This file is part of vsc-utils,
55
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -35,12 +35,12 @@
3535
import random
3636
import string
3737
from pwd import getpwuid
38+
from io import StringIO
3839

3940
from vsc.install.testing import TestCase
4041

4142
from vsc.utils.nagios import NagiosReporter, SimpleNagios
4243
from vsc.utils.nagios import NAGIOS_EXIT_OK, NAGIOS_EXIT_WARNING, NAGIOS_EXIT_CRITICAL, NAGIOS_EXIT_UNKNOWN
43-
from vsc.utils.py2vs3 import StringIO
4444

4545

4646
class TestNagios(TestCase):

‎test/nagios_results.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# encoding: utf-8
22
#
3-
# Copyright 2012-2021 Ghent University
3+
# Copyright 2012-2023 Ghent University
44
#
55
# This file is part of vsc-utils,
66
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

0 commit comments

Comments
 (0)