From 9edfbd4e73ae3faf2aa40680289f81b96c674ea6 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 8 Oct 2025 12:52:39 +0200 Subject: [PATCH 1/2] feat: ruff from vsc-install --- Jenkinsfile | 12 ++++++++++++ ruff.toml | 15 +++++++++++++++ vsc-ci.ini | 2 ++ 3 files changed, 29 insertions(+) create mode 100644 ruff.toml diff --git a/Jenkinsfile b/Jenkinsfile index 57fd0ef..755b724 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,18 @@ node { // remove untracked files (*.pyc for example) sh 'git clean -fxd' } + stage ('ruff format') { + sh 'curl -L --silent https://github.com/astral-sh/ruff/releases/download/0.13.1/ruff-x86_64-unknown-linux-gnu.tar.gz --output - | tar -xzv' + sh 'cp ruff-x86_64-unknown-linux-gnu/ruff .' + sh './ruff --version' + sh './ruff format --check .' + } + stage ('ruff check') { + sh 'curl -L --silent https://github.com/astral-sh/ruff/releases/download/0.13.1/ruff-x86_64-unknown-linux-gnu.tar.gz --output - | tar -xzv' + sh 'cp ruff-x86_64-unknown-linux-gnu/ruff .' + sh './ruff --version' + sh './ruff check .' + } stage('test') { sh 'pip3 install --ignore-installed --prefix $PWD/.vsc-tox tox' sh 'export PATH=$PWD/.vsc-tox/bin:$PATH && export PYTHONPATH=$PWD/.vsc-tox/lib/python$(python3 -c "import sys; print(\\"%s.%s\\" % sys.version_info[:2])")/site-packages:$PYTHONPATH && tox -v -c tox.ini' diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..3df3c78 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,15 @@ +line-length = 120 +indent-width = 4 +preview = true +[lint] +extend-select = ['E101', 'E501', 'E713', 'E4', 'E7', 'E9', 'F', 'F811', 'W291', 'PLR0911', 'PLW0602', 'PLW0604', 'PLW0108', 'PLW0127', 'PLW0129', 'PLW1501', 'PLR0124', 'PLR0202', 'PLR0203', 'PLR0402', 'PLR0913', 'B028', 'B905', 'C402', 'C403', 'UP032', 'UP037', 'UP025', 'UP036', 'UP034', 'UP033', 'UP031', 'UP004'] +exclude = ['.bzr', '.direnv', '.eggs', '.git', '.git-rewrite', '.hg', '.ipynb_checkpoints', '.mypy_cache', '.nox', '.pants.d', '.pyenv', '.pytest_cache', '.pytype', '.ruff_cache', '.svn', '.tox', '.venv', '.vscode', '__pypackages__', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'site-packages', 'venv', 'test/prospectortest/*'] +ignore = ['E731'] +pylint.max-args = 11 +[format] +quote-style = "double" +indent-style = "space" +docstring-code-format = true +docstring-code-line-length = 120 +line-ending = "lf" +exclude = ["test/prospectortest/*"] diff --git a/vsc-ci.ini b/vsc-ci.ini index ebb0e46..930038e 100644 --- a/vsc-ci.ini +++ b/vsc-ci.ini @@ -1,2 +1,4 @@ [vsc-ci] py39_tests_must_pass=1 +run_ruff_format_check=1 +run_ruff_check=1 From ca4687221541a0042cf4731c9d940066ab4e9d78 Mon Sep 17 00:00:00 2001 From: Andy Georges Date: Wed, 8 Oct 2025 17:50:16 +0200 Subject: [PATCH 2/2] ruff: fix tests --- lib/vsc/__init__.py | 5 +- lib/vsc/filesystem/__init__.py | 2 + lib/vsc/filesystem/ext.py | 4 +- lib/vsc/filesystem/gpfs.py | 459 +++++++++++++++++++-------------- lib/vsc/filesystem/lustre.py | 331 +++++++++++++----------- lib/vsc/filesystem/operator.py | 10 +- lib/vsc/filesystem/posix.py | 151 ++++++----- ruff.toml | 4 +- setup.py | 6 +- 9 files changed, 548 insertions(+), 424 deletions(-) diff --git a/lib/vsc/__init__.py b/lib/vsc/__init__.py index 0b86be9..132586b 100644 --- a/lib/vsc/__init__.py +++ b/lib/vsc/__init__.py @@ -17,7 +17,8 @@ @author: Andy Georges (Ghent University) """ -#the vsc namespace is used in different folders allong the system -#so explicitly declare this is also the vsc namespace +# the vsc namespace is used in different folders allong the system +# so explicitly declare this is also the vsc namespace import pkg_resources + pkg_resources.declare_namespace(__name__) diff --git a/lib/vsc/filesystem/__init__.py b/lib/vsc/filesystem/__init__.py index 0f931d3..1f91944 100644 --- a/lib/vsc/filesystem/__init__.py +++ b/lib/vsc/filesystem/__init__.py @@ -17,6 +17,8 @@ @author: Stijn De Weirdt (Ghent University) """ + # Allow other packages to extend this namespace, zip safe setuptools style import pkg_resources + pkg_resources.declare_namespace(__name__) diff --git a/lib/vsc/filesystem/ext.py b/lib/vsc/filesystem/ext.py index a0fdcda..cfd5015 100644 --- a/lib/vsc/filesystem/ext.py +++ b/lib/vsc/filesystem/ext.py @@ -23,12 +23,12 @@ from vsc.filesystem.posix import PosixOperations, PosixOperationError + class ExtOperationError(PosixOperationError): pass class ExtOperations(PosixOperations): - def __init__(self): super().__init__() - self.supportedfilesystems = ['ext2', 'ext3', 'ext4'] + self.supportedfilesystems = ["ext2", "ext3", "ext4"] diff --git a/lib/vsc/filesystem/gpfs.py b/lib/vsc/filesystem/gpfs.py index 67db120..3282787 100644 --- a/lib/vsc/filesystem/gpfs.py +++ b/lib/vsc/filesystem/gpfs.py @@ -18,6 +18,7 @@ @author: Stijn De Weirdt (Ghent University) @author: Andy Georges (Ghent University) """ + import copy import os import re @@ -26,7 +27,7 @@ from socket import gethostname from itertools import dropwhile from enum import Enum -from urllib.parse import unquote as percentdecode +from urllib.parse import unquote as percentdecode from vsc.config.base import DEFAULT_INODE_MAX, DEFAULT_INODE_PREALLOC from vsc.filesystem.posix import PosixOperations, PosixOperationError @@ -34,24 +35,42 @@ from vsc.utils.missing import nub, find_sublist_index, RUDict from vsc.utils.patterns import Singleton -GPFS_BIN_PATH = '/usr/lpp/mmfs/bin' +GPFS_BIN_PATH = "/usr/lpp/mmfs/bin" GPFS_DEFAULT_INODE_LIMIT = f"{int(DEFAULT_INODE_MAX)}:{int(DEFAULT_INODE_PREALLOC)}" -StorageQuota = namedtuple('StorageQuota', - ['name', - 'blockUsage', 'blockQuota', 'blockLimit', 'blockInDoubt', 'blockGrace', - 'filesUsage', 'filesQuota', 'filesLimit', 'filesInDoubt', 'filesGrace', - 'remarks', 'quota', 'defQuota', 'fid', 'filesetname']) +StorageQuota = namedtuple( + "StorageQuota", + [ + "name", + "blockUsage", + "blockQuota", + "blockLimit", + "blockInDoubt", + "blockGrace", + "filesUsage", + "filesQuota", + "filesLimit", + "filesInDoubt", + "filesGrace", + "remarks", + "quota", + "defQuota", + "fid", + "filesetname", + ], +) + class Typ2Param(Enum): - USR = 'USR' - GRP = 'GRP' - FILESET = 'FILESET' - -GPFS_OK_STATES = ['HEALTHY', 'DISABLED', 'TIPS'] -GPFS_WARNING_STATES = ['DEGRADED'] -GPFS_ERROR_STATES = ['FAILED', 'DEPEND'] -GPFS_UNKNOWN_STATES = ['CHECKING', 'UNKNOWN'] + USR = "USR" + GRP = "GRP" + FILESET = "FILESET" + + +GPFS_OK_STATES = ["HEALTHY", "DISABLED", "TIPS"] +GPFS_WARNING_STATES = ["DEGRADED"] +GPFS_ERROR_STATES = ["FAILED", "DEPEND"] +GPFS_UNKNOWN_STATES = ["CHECKING", "UNKNOWN"] GPFS_HEALTH_STATES = GPFS_OK_STATES + GPFS_WARNING_STATES + GPFS_ERROR_STATES + GPFS_UNKNOWN_STATES GPFS_NOGRACE_REGEX = re.compile(r"none", re.I) @@ -64,7 +83,7 @@ def _automatic_mount_only(fs): """ Filter that returns true if the filesystem is automount enabled """ - return fs['automaticMountOption'] in ('yes', 'automount') + return fs["automaticMountOption"] in ("yes", "automount") def split_output_lines(out): @@ -82,7 +101,7 @@ def clean(line): else: return line - return [[percentdecode(y) for y in clean(x).split(':')] for x in out] + return [[percentdecode(y) for y in clean(x).split(":")] for x in out] class GpfsOperationError(PosixOperationError): @@ -90,10 +109,9 @@ class GpfsOperationError(PosixOperationError): class GpfsOperations(PosixOperations, metaclass=Singleton): - def __init__(self): super().__init__() - self.supportedfilesystems = ['gpfs', 'nfs'] + self.supportedfilesystems = ["gpfs", "nfs"] self.gpfslocalfilesystems = None # the locally found GPFS filesystems self.gpfslocalquotas = None @@ -106,7 +124,7 @@ def __init__(self): # pylint: disable=arguments-differ def _execute(self, name, opts=None, changes=False): """Return and check the GPFS command. - @type cmd: string, will be prefixed by GPFS_BIN_PATH if not absolute + @type cmd: string, will be prefixed by GPFS_BIN_PATH if not absolute """ if os.path.isabs(name): @@ -117,12 +135,18 @@ def _execute(self, name, opts=None, changes=False): cmd = [cmdname] if opts is not None: - if isinstance(opts, (tuple, list,)): + if isinstance( + opts, + ( + tuple, + list, + ), + ): cmd += list(opts) else: self.log.raiseException( - f"_execute: please use a list or tuple for options: cmd {cmdname} opts {opts}", - GpfsOperationError) + f"_execute: please use a list or tuple for options: cmd {cmdname} opts {opts}", GpfsOperationError + ) ec, out = super()._execute(cmd, changes) @@ -135,21 +159,22 @@ def _local_filesystems(self): if self.gpfslocalfilesystems is None: self.list_filesystems() - self.localfilesystemnaming.append('gpfsdevice') + self.localfilesystemnaming.append("gpfsdevice") for fs in self.localfilesystems: - if fs[self.localfilesystemnaming.index('type')] == 'gpfs': - localdevice = fs[self.localfilesystemnaming.index('device')] + if fs[self.localfilesystemnaming.index("type")] == "gpfs": + localdevice = fs[self.localfilesystemnaming.index("device")] - expectedprefix = '/dev' + expectedprefix = "/dev" if localdevice.startswith(expectedprefix): - tmp = localdevice.split(os.sep)[len(expectedprefix.split(os.sep)):] + tmp = localdevice.split(os.sep)[len(expectedprefix.split(os.sep)) :] if len(tmp) == 1: gpfsdevice = tmp[0] else: fs.append(None) self.log.raiseException( f"Something went wrong trying to resolve GPFS device from localfilesystem device: fs {fs}", - GpfsOperationError) + GpfsOperationError, + ) else: gpfsdevice = localdevice @@ -157,9 +182,15 @@ def _local_filesystems(self): fs.append(gpfsdevice) else: fs.append(None) - self.log.warning(("While trying to resolve GPFS device from localfilesystem device" - " fs %s found gpfsdevice %s that is not in gpfslocalfilesystems %s"), - fs, gpfsdevice, self.gpfslocalfilesystems.keys()) + self.log.warning( + ( + "While trying to resolve GPFS device from localfilesystem device" + " fs %s found gpfsdevice %s that is not in gpfslocalfilesystems %s" + ), + fs, + gpfsdevice, + self.gpfslocalfilesystems.keys(), + ) else: fs.append(None) @@ -184,7 +215,8 @@ def fixup_executeY_line(self, fields, description_count): if sub_index is None: self.log.raiseException( f"Too many fields: {len(fields)} (description has {description_count} fields)." - f"Cannot find match for the start field. Not fixing line {fields}") + f"Cannot find match for the start field. Not fixing line {fields}" + ) else: self.log.info("Fixing found an index for the sublist at %d", sub_index) line = expected_start_fields + ls[:sub_index] @@ -192,54 +224,65 @@ def fixup_executeY_line(self, fields, description_count): if len(line) > description_count: self.log.raiseException( - f"After fixing, line still has too many fields: line ({line}), original ({fields})") + f"After fixing, line still has too many fields: line ({line}), original ({fields})" + ) # now we need to check if the string in the first field has somehow magically merged with the previous line first_field = fields[0] if remainder[0] == first_field: - line.extend([''] * (description_count - len(line))) + line.extend([""] * (description_count - len(line))) return [line, remainder] elif line[-1].endswith(first_field): line[-1] = line[-1].rstrip(first_field) - line.extend([''] * (description_count - len(line))) + line.extend([""] * (description_count - len(line))) remainder.insert(0, first_field) return [line, remainder] else: self.log.raiseException( f"Failed to find the initial field of the line: {first_field} " - f"after fixup and splitting line into [{line}, {remainder}]") + f"after fixup and splitting line into [{line}, {remainder}]" + ) return [] def _assemble_fields(self, fields, out): - """Assemble executeY output fields """ + """Assemble executeY output fields""" # do we have multiple field counts? field_counts = [i for (i, _) in fields] if len(nub(field_counts)) > 1: maximum_field_count = max(field_counts) description_field_count = field_counts[0] - for (field_count, line) in fields[1:]: + for field_count, line in fields[1:]: if field_count == description_field_count: continue elif field_count < description_field_count: - self.log.debug("Description length %s greater then %s. Adding whitespace. (names %s, row %s)", - maximum_field_count, field_count, fields[0][6:], line[6:]) - line.extend([''] * (maximum_field_count - field_count)) + self.log.debug( + "Description length %s greater then %s. Adding whitespace. (names %s, row %s)", + maximum_field_count, + field_count, + fields[0][6:], + line[6:], + ) + line.extend([""] * (maximum_field_count - field_count)) else: # try to fix the line - self.log.info("Line has too many fields (%d > %d), trying to fix %s", - field_count, description_field_count, line) + self.log.info( + "Line has too many fields (%d > %d), trying to fix %s", + field_count, + description_field_count, + line, + ) fixed_lines = self.fixup_executeY_line(line, description_field_count) i = fields.index((field_count, line)) - fields[i:i + 1] = map(lambda fs: (len(fs), fs), fixed_lines) + fields[i : i + 1] = map(lambda fs: (len(fs), fs), fixed_lines) # assemble result res = defaultdict(list) try: for index, name in enumerate(fields[0][1][6:]): - if name != '': - for (_, line) in fields[1:]: + if name != "": + for _, line in fields[1:]: res[name].append(line[6 + index]) except IndexError: self.log.raiseException(f"Failed to regroup data {fields} (from output {out})") @@ -248,20 +291,26 @@ def _assemble_fields(self, fields, out): def _executeY(self, name, opts=None, prefix=False): """Run with -Y and parse output in dict of name:list of values - type prefix: boolean, if true prefix the -Y to the options (otherwise append the option). + type prefix: boolean, if true prefix the -Y to the options (otherwise append the option). """ if opts is None: opts = [] - elif isinstance(opts, (tuple, list,)): + elif isinstance( + opts, + ( + tuple, + list, + ), + ): opts = list(opts) else: self.log.error("_executeY: have to use a list or tuple for options: name %s opts %s", name, opts) return {} if prefix: - opts.insert(0, '-Y') + opts.insert(0, "-Y") else: - opts.append('-Y') + opts.append("-Y") _, out = self._execute(name, opts) @@ -275,7 +324,7 @@ def _executeY(self, name, opts=None, prefix=False): b = [[percentdecode(y) for y in x.split(':')] for x in a] """ what = split_output_lines(out.splitlines()) - expectedheader = [name, '', 'HEADER', 'version', 'reserved', 'reserved'] + expectedheader = [name, "", "HEADER", "version", "reserved", "reserved"] # verify result and remove all items that do not match the expected output data # e.g. mmrepquota start with single line of unnecessary ouput (which may be repeated for USR, GRP and FILESET) @@ -288,9 +337,9 @@ def _executeY(self, name, opts=None, prefix=False): return self._assemble_fields(fields, out) else: # mmhealth command has other header, no other known command does this. - self.log.info('Not the default header, trying state and event headers') + self.log.info("Not the default header, trying state and event headers") rest = {} - for typ in ('State', 'Event'): + for typ in ("State", "Event"): try: fields = [(len(x), x) for x in what if x[1] == typ] except IndexError: @@ -300,12 +349,13 @@ def _executeY(self, name, opts=None, prefix=False): rest[typ] = res else: - self.log.raiseException(f"No valid lines of header type {typ} for output: {out}", - GpfsOperationError) + self.log.raiseException( + f"No valid lines of header type {typ} for output: {out}", GpfsOperationError + ) return rest - def list_filesystems(self, device='all', update=False, fs_filter=_automatic_mount_only): + def list_filesystems(self, device="all", update=False, fs_filter=_automatic_mount_only): """ List all filesystems. @@ -324,12 +374,11 @@ def list_filesystems(self, device='all', update=False, fs_filter=_automatic_moun res = RUDict() for device in devices: - - info = self._executeY('mmlsfs', [device]) + info = self._executeY("mmlsfs", [device]) # for v3.5 deviceName:fieldName:data:remarks: # set the gpfsdevices - gpfsdevices = nub(info.get('deviceName', [])) + gpfsdevices = nub(info.get("deviceName", [])) if len(gpfsdevices) == 0: self.log.raiseException(f"No devices found. Returned info {info}", GpfsOperationError) else: @@ -337,7 +386,7 @@ def list_filesystems(self, device='all', update=False, fs_filter=_automatic_moun res_ = {dev: {} for dev in gpfsdevices} # build structure res.update(res_) - for dev, k, v in zip(info['deviceName'], info['fieldName'], info['data']): + for dev, k, v in zip(info["deviceName"], info["fieldName"], info["data"]): res[dev][k] = v if fs_filter: @@ -384,28 +433,28 @@ def list_quota(self, devices=None): info = defaultdict(list) for device in devices: - res = self._executeY('mmrepquota', ['-n', device], prefix=True) - for (key, value) in res.items(): + res = self._executeY("mmrepquota", ["-n", device], prefix=True) + for key, value in res.items(): info[key].extend(value) datakeys = list(info.keys()) - datakeys.remove('filesystemName') - datakeys.remove('quotaType') - datakeys.remove('id') + datakeys.remove("filesystemName") + datakeys.remove("quotaType") + datakeys.remove("id") - fss = nub(info.get('filesystemName', [])) + fss = nub(info.get("filesystemName", [])) self.log.debug("Found the following filesystem names: %s", fss) - quotatypes = nub(info.get('quotaType', [])) + quotatypes = nub(info.get("quotaType", [])) quotatypesstruct = {qt: defaultdict(list) for qt in quotatypes} res = {fs: copy.deepcopy(quotatypesstruct) for fs in fss} # build structure - for idx, (fs, qt, qid) in enumerate(zip(info['filesystemName'], info['quotaType'], info['id'])): + for idx, (fs, qt, qid) in enumerate(zip(info["filesystemName"], info["quotaType"], info["id"])): details = {k: info[k][idx] for k in datakeys} if qt == self.quota_types.FILESET.value: # GPFS fileset quota have empty filesetName field - details['filesetname'] = details['name'] + details["filesetname"] = details["name"] res[fs][qt][qid].append(StorageQuota(**details)) self.gpfslocalquotas = res @@ -443,7 +492,7 @@ def list_filesets(self, devices=None, filesetnames=None, update=False): if isinstance(filesetnames, str): filesetnames = [filesetnames] - filesetnamestxt = ','.join(filesetnames) + filesetnamestxt = ",".join(filesetnames) opts.append(filesetnamestxt) self.log.debug("Looking up filesets for devices %s", devices) @@ -452,7 +501,7 @@ def list_filesets(self, devices=None, filesetnames=None, update=False): for device in devices: opts_ = copy.deepcopy(opts) opts_.insert(1, device) - res = self._executeY('mmlsfileset', opts_) + res = self._executeY("mmlsfileset", opts_) # for v3.5 # filesystemName:filesetName:id:rootInode:status:path:parentId:created:inodes:dataInKB:comment: # filesetMode:afmTarget:afmState:afmMode:afmFileLookupRefreshInterval:afmFileOpenRefreshInterval: @@ -461,18 +510,18 @@ def list_filesets(self, devices=None, filesetnames=None, update=False): # afmNumReadThreads:afmNumReadGWs:afmReadBufferSize:afmWriteBufferSize:afmReadSparseThreshold: # afmParallelReadChunkSize:afmParallelReadThreshold:snapId: self.log.debug("list_filesets res keys = %s ", res.keys()) - for (key, value) in res.items(): + for key, value in res.items(): info[key].extend(value) datakeys = list(info.keys()) - datakeys.remove('filesystemName') - datakeys.remove('id') + datakeys.remove("filesystemName") + datakeys.remove("id") - fss = nub(info.get('filesystemName', [])) + fss = nub(info.get("filesystemName", [])) res = {fs: {} for fs in fss} # build structure - count = len(info['filesystemName']) - for idx, (fs, qid) in enumerate(zip(info['filesystemName'], info['id'])): + count = len(info["filesystemName"]) + for idx, (fs, qid) in enumerate(zip(info["filesystemName"], info["id"])): try: self.log.debug(f"Getting details for {idx} {fs} {qid}") details = {k: info[k][idx] for k in datakeys if len(info[k]) == count} @@ -518,10 +567,11 @@ def get_fileset_info(self, filesystem_name, fileset_name): filesets = self.gpfslocalfilesets[filesystem_name] except KeyError: self.log.raiseException( - f"GPFS has no fileset information for filesystem {filesystem_name}", GpfsOperationError) + f"GPFS has no fileset information for filesystem {filesystem_name}", GpfsOperationError + ) for fset in filesets.values(): - if fset['filesetName'] == fileset_name: + if fset["filesetName"] == fileset_name: return fset return None @@ -536,7 +586,7 @@ def get_fileset_name(self, fileset_id, filesystem_name): self.list_filesets(devices=filesystem_name) try: - fileset_name = self.gpfslocalfilesets[filesystem_name][fileset_id]['filesetName'] + fileset_name = self.gpfslocalfilesets[filesystem_name][fileset_id]["filesetName"] except KeyError: errmsg = f"Fileset ID '{fileset_id}' not found in GPFS filesystem '{filesystem_name}'" self.log.raiseException(errmsg, GpfsOperationError) @@ -580,16 +630,16 @@ def get_quota_owner(self, quota_id, filesystem_name): def _list_disk_single_device(self, device): """Return disk info for specific device - both -M and -L info + both -M and -L info """ - shorthn = gethostname().split('.')[0] + shorthn = gethostname().split(".")[0] - infoL = self._executeY('mmlsdisk', [device, '-L']) + infoL = self._executeY("mmlsdisk", [device, "-L"]) keysL = list(infoL.keys()) - keysL.remove('nsdName') - infoM = self._executeY('mmlsdisk', [device, '-M']) + keysL.remove("nsdName") + infoM = self._executeY("mmlsdisk", [device, "-M"]) keysM = list(infoM.keys()) - keysM.remove('nsdName') + keysM.remove("nsdName") # sanity check @@ -597,20 +647,20 @@ def _list_disk_single_device(self, device): # if this fails, nodes probably have shortnames try: # - means disk offline, so no nodename - alldomains = ['.'.join(x.split('.')[1:]) for x in infoM['IOPerformedOnNode'] if x not in ['-', 'localhost']] + alldomains = [".".join(x.split(".")[1:]) for x in infoM["IOPerformedOnNode"] if x not in ["-", "localhost"]] if len(set(alldomains)) > 1: self.log.error("More than one domain found: %s.", alldomains) commondomain = alldomains[0] # TODO: should be most frequent one except (IndexError, KeyError): - self.log.exception("Cannot determine domainname for nodes %s", infoM['IOPerformedOnNode']) + self.log.exception("Cannot determine domainname for nodes %s", infoM["IOPerformedOnNode"]) commondomain = None - for idx, node in enumerate(infoM['IOPerformedOnNode']): - if node == 'localhost': - infoM['IOPerformedOnNode'][idx] = '.'.join([x for x in [shorthn, commondomain] if x is not None]) + for idx, node in enumerate(infoM["IOPerformedOnNode"]): + if node == "localhost": + infoM["IOPerformedOnNode"][idx] = ".".join([x for x in [shorthn, commondomain] if x is not None]) - res = {nsd: {} for nsd in infoL['nsdName']} # build structure - for idx, nsd in enumerate(infoL['nsdName']): + res = {nsd: {} for nsd in infoL["nsdName"]} # build structure + for idx, nsd in enumerate(infoL["nsdName"]): for k in keysL: res[nsd][k] = infoL[k][idx] for k in keysM: @@ -618,8 +668,13 @@ def _list_disk_single_device(self, device): if k in keysL: # duplicate key !! if not infoL[k][idx] == infoM[k][idx]: - self.log.error(("nsdName %s has named value %s in both -L and -M, but have different value" - " L=%s M=%s"), nsd, k, infoL[k][idx], infoM[k][idx]) + self.log.error( + ("nsdName %s has named value %s in both -L and -M, but have different value L=%s M=%s"), + nsd, + k, + infoL[k][idx], + infoM[k][idx], + ) Mk = f"M_{k}" res[nsd][Mk] = infoM[k][idx] @@ -627,9 +682,9 @@ def _list_disk_single_device(self, device): def list_disks(self, devices=None): """List all disks for devices (if devices is None, use all devices - Return dict with - key = device values is dict - key is disk, value is remaining property + Return dict with + key = device values is dict + key is disk, value is remaining property """ if devices is None: # get all devices from all filesystems @@ -650,7 +705,7 @@ def list_disks(self, devices=None): def list_nsds(self): """List NSD info - Not implemented due to missing -Y option of mmlsnsd + Not implemented due to missing -Y option of mmlsnsd """ self.log.error("listNsds not implemented.") @@ -663,47 +718,54 @@ def getAttr(self, obj=None): if not self.exists(obj): self.log.raiseException("getAttr: obj %s does not exist", GpfsOperationError) - ec, out = self._execute('mmlsattr', ["-L", obj]) + ec, out = self._execute("mmlsattr", ["-L", obj]) if ec > 0: self.log.raiseException(f"getAttr: mmlsattr with opts -L {obj} failed", GpfsOperationError) res = {} for line in out.split("\n"): - line = re.sub(r"\s+", '', line) + line = re.sub(r"\s+", "", line) if len(line) == 0: continue items = line.split(":") if len(items) == 1: - items.append('') # fix anomalies + items.append("") # fix anomalies # creationtime has : in value as well eg creationtime:ThuAug2313:04:202012 res[items[0]] = ":".join(items[1:]) return res def get_details(self, obj=None): - """Given obj, return as much relevant info as possible - """ + """Given obj, return as much relevant info as possible""" obj = self._sanity_check(obj) - res = {'parent': None} - res['exists'] = self.exists(obj) + res = {"parent": None} + res["exists"] = self.exists(obj) - if res['exists']: + if res["exists"]: realpath = obj else: realpath = self._largest_existing_path(obj) - res['parent'] = realpath + res["parent"] = realpath fs = self._what_filesystem(obj) - res['fs'] = fs + res["fs"] = fs - res['attrs'] = self.getAttr(obj) + res["attrs"] = self.getAttr(obj) return res - def make_fileset(self, new_fileset_path, fileset_name=None, parent_fileset_name=None, afm=None, inodes_max=None, - inodes_prealloc=None, fileset_id=None): + def make_fileset( + self, + new_fileset_path, + fileset_name=None, + parent_fileset_name=None, + afm=None, + inodes_max=None, + inodes_prealloc=None, + fileset_id=None, + ): """ Given path, create a new fileset and link it to said path - check uniqueness @@ -757,78 +819,84 @@ def make_fileset(self, new_fileset_path, fileset_name=None, parent_fileset_name= self.log.raiseException( f"makeFileset for new_fileset_path {new_fileset_path} returned sane fsetpath {fsetpath}," " but it already exists.", - GpfsOperationError) + GpfsOperationError, + ) # choose unique name parentfsetpath = os.path.dirname(fsetpath) if not self.exists(parentfsetpath): self.log.raiseException( f"parent dir {parentfsetpath} of fsetpath {fsetpath} does not exist." - "Not going to create it automatically.", - GpfsOperationError) + "Not going to create it automatically.", + GpfsOperationError, + ) fs = self.what_filesystem(parentfsetpath) - foundgpfsdevice = fs[self.localfilesystemnaming.index('gpfsdevice')] + foundgpfsdevice = fs[self.localfilesystemnaming.index("gpfsdevice")] # FIXME: Not sure if this is a good idea. if fileset_name is None: # guess the device from the pathname # subtract the device mount path from filesetpath ? (what with filesets in filesets) - mntpt = fs[self.localfilesystemnaming.index('mountpoint')] + mntpt = fs[self.localfilesystemnaming.index("mountpoint")] if fsetpath.startswith(mntpt): - lastpart = fsetpath.split(os.sep)[len(mntpt.split(os.sep)):] + lastpart = fsetpath.split(os.sep)[len(mntpt.split(os.sep)) :] fileset_name = "_".join(lastpart) else: fileset_name = os.path.basedir(fsetpath) - self.log.error("fsetpath %s doesn't start with mntpt %s. using basedir %s", - fsetpath, mntpt, fileset_name) + self.log.error( + "fsetpath %s doesn't start with mntpt %s. using basedir %s", fsetpath, mntpt, fileset_name + ) # bail if there is a fileset with the same name or the same link location, i.e., path for efset in self.gpfslocalfilesets[foundgpfsdevice].values(): - efsetpath = efset.get('path', None) - efsetname = efset.get('filesetName', None) + efsetpath = efset.get("path", None) + efsetname = efset.get("filesetName", None) if efsetpath == fsetpath or efsetname == fileset_name: self.log.raiseException( f"Found existing fileset {efset} that has same path {efsetpath} or same name {efsetname}" f" as new path {fsetpath} or new name {fileset_name}", - GpfsOperationError) + GpfsOperationError, + ) # create the fileset # if created, try to link it with -J to path mmcrfileset_options = [foundgpfsdevice, fileset_name] if parent_fileset_name is None: - mmcrfileset_options += ['--inode-space', 'new'] + mmcrfileset_options += ["--inode-space", "new"] if inodes_max: INODE_LIMIT_STRING = f"{int(inodes_max)}" if inodes_prealloc: INODE_LIMIT_STRING += f":{int(inodes_prealloc)}" else: INODE_LIMIT_STRING = GPFS_DEFAULT_INODE_LIMIT - mmcrfileset_options += ['--inode-limit', INODE_LIMIT_STRING] + mmcrfileset_options += ["--inode-limit", INODE_LIMIT_STRING] else: parent_fileset_exists = False for efset in self.gpfslocalfilesets[foundgpfsdevice].values(): - if parent_fileset_name and parent_fileset_name == efset.get('filesetName', None): + if parent_fileset_name and parent_fileset_name == efset.get("filesetName", None): parent_fileset_exists = True if not parent_fileset_exists: self.log.raiseException( - f"Parent fileset {parent_fileset_name} does not appear to exist.", - GpfsOperationError) - mmcrfileset_options += ['--inode-space', parent_fileset_name] + f"Parent fileset {parent_fileset_name} does not appear to exist.", GpfsOperationError + ) + mmcrfileset_options += ["--inode-space", parent_fileset_name] - (ec, out) = self._execute('mmcrfileset', mmcrfileset_options, True) + (ec, out) = self._execute("mmcrfileset", mmcrfileset_options, True) if ec > 0: self.log.raiseException( f"Creating fileset with name {fileset_name} on device {foundgpfsdevice} failed (out: {out})", - GpfsOperationError) + GpfsOperationError, + ) # link the fileset - ec, out = self._execute('mmlinkfileset', [foundgpfsdevice, fileset_name, '-J', fsetpath], True) + ec, out = self._execute("mmlinkfileset", [foundgpfsdevice, fileset_name, "-J", fsetpath], True) if ec > 0: self.log.raiseException( f"Linking fileset with name {fileset_name} on device {foundgpfsdevice} " f"to path {fsetpath} failed (out: {out})", - GpfsOperationError) + GpfsOperationError, + ) # at the end, rescan the filesets and force update the info self.list_filesets(update=True) @@ -842,7 +910,7 @@ def set_user_quota(self, soft, user, obj=None, hard=None, inode_soft=None, inode @type inode_soft: integer representing the soft files limit @type inode_soft: integer representing the hard files quota """ - self._set_quota(soft, who=user, obj=obj, typ='user', hard=hard, inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota(soft, who=user, obj=obj, typ="user", hard=hard, inode_soft=inode_soft, inode_hard=inode_hard) def set_group_quota(self, soft, group, obj=None, hard=None, inode_soft=None, inode_hard=None): """Set quota for a group on a given object (e.g., a path in the filesystem, which may correpond to a fileset) @@ -855,7 +923,7 @@ def set_group_quota(self, soft, group, obj=None, hard=None, inode_soft=None, ino @type inode_soft: integer representing the soft files limit @type inode_soft: integer representing the hard files quota """ - self._set_quota(soft, who=group, obj=obj, typ='group', hard=hard, inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota(soft, who=group, obj=obj, typ="group", hard=hard, inode_soft=inode_soft, inode_hard=inode_hard) def set_fileset_quota(self, soft, fileset_path, fileset_name=None, hard=None, inode_soft=None, inode_hard=None): """Set quota on a fileset. @@ -870,16 +938,24 @@ def set_fileset_quota(self, soft, fileset_path, fileset_name=None, hard=None, in # we need the corresponding fileset name if fileset_name is None: attr = self.getAttr(fileset_path) - if 'filesetname' in attr: - fileset_name = attr['filesetname'] + if "filesetname" in attr: + fileset_name = attr["filesetname"] self.log.info("set_fileset_quota: setting fileset to %s for obj %s", fileset_name, fileset_path) else: self.log.raiseException( f"set_fileset_quota: attrs for obj {fileset_path} don't have filestename property (attr: {attr})", - GpfsOperationError) + GpfsOperationError, + ) - self._set_quota(soft, who=fileset_name, obj=fileset_path, typ='fileset', hard=hard, - inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota( + soft, + who=fileset_name, + obj=fileset_path, + typ="fileset", + hard=hard, + inode_soft=inode_soft, + inode_hard=inode_hard, + ) def set_user_grace(self, obj, grace=0, who=None): """Set the grace period for user data. @@ -889,7 +965,7 @@ def set_user_grace(self, obj, grace=0, who=None): @type who: identifier (eg username or UID) """ _ = who # avoid lint error, unused in GPFS as user is determined from obj ownership - self._set_grace(obj, 'user', grace) + self._set_grace(obj, "user", grace) def set_group_grace(self, obj, grace=0, who=None): """Set the grace period for user data. @@ -899,7 +975,7 @@ def set_group_grace(self, obj, grace=0, who=None): @type who: identifier (eg group name or GID) """ _ = who # avoid lint error, unused in GPFS as group is determined from obj ownership - self._set_grace(obj, 'group', grace) + self._set_grace(obj, "group", grace) def set_fileset_grace(self, obj, grace=0): """Set the grace period for fileset data. @@ -907,7 +983,7 @@ def set_fileset_grace(self, obj, grace=0): @type obj: string representing the path where the GPFS was mounted or the device itself @type grace: grace period expressed in seconds """ - self._set_grace(obj, 'fileset', grace) + self._set_grace(obj, "fileset", grace) def _set_grace(self, obj, typ, grace=0, id_=0): """Set the grace period for a given type of objects in GPFS. @@ -922,10 +998,11 @@ def _set_grace(self, obj, typ, grace=0, id_=0): self.log.raiseException(f"setQuota: can't set quota on none-existing obj {obj}", GpfsOperationError) # FIXME: this should be some constant or such - typ2opt = {'user': 'u', - 'group': 'g', - 'fileset': 'j', - } + typ2opt = { + "user": "u", + "group": "g", + "fileset": "j", + } opts = [] opts += [f"-{typ2opt[typ]}", f"{id_}"] @@ -933,7 +1010,7 @@ def _set_grace(self, obj, typ, grace=0, id_=0): opts.append(obj) - ec, _ = self._execute('tssetquota', opts, True) + ec, _ = self._execute("tssetquota", opts, True) if ec > 0: self.log.raiseException(f"_set_grace: tssetquota with opts {opts} failed", GpfsOperationError) @@ -970,13 +1047,13 @@ def _get_grace_expiration(grace_record): elif grace: grace = grace.groupdict() grace_time = 0 - if grace['days']: - grace_time = int(grace['days']) * 86400 - elif grace['hours']: - grace_time = int(grace['hours']) * 3600 - elif grace['minutes']: - grace_time = int(grace['minutes']) * 60 - elif grace['expired']: + if grace["days"]: + grace_time = int(grace["days"]) * 86400 + elif grace["hours"]: + grace_time = int(grace["hours"]) * 3600 + elif grace["minutes"]: + grace_time = int(grace["minutes"]) * 60 + elif grace["expired"]: grace_time = 0 else: errmsg = "Unprocessed grace groupdict %s (from string %s)." @@ -989,7 +1066,7 @@ def _get_grace_expiration(grace_record): return expired - def _set_quota(self, soft, who, obj=None, typ='user', hard=None, inode_soft=None, inode_hard=None): + def _set_quota(self, soft, who, obj=None, typ="user", hard=None, inode_soft=None, inode_hard=None): """Set quota on the given object. @type soft: integer representing the soft limit expressed in bytes @@ -1032,9 +1109,9 @@ def _set_quota(self, soft, who, obj=None, typ='user', hard=None, inode_soft=None # FIXME: this should be some constant or such typ2opt = { - 'user': 'u', - 'group': 'g', - 'fileset': 'j', + "user": "u", + "group": "g", + "fileset": "j", } soft2hard_factor = 1.05 @@ -1048,12 +1125,12 @@ def _set_quota(self, soft, who, obj=None, typ='user', hard=None, inode_soft=None hard = int(soft * soft2hard_factor) elif hard < soft: self.log.raiseException( - f"setQuota: can't set hard limit {hard} lower then soft limit {soft}", - GpfsOperationError) + f"setQuota: can't set hard limit {hard} lower then soft limit {soft}", GpfsOperationError + ) opts += [f"-{typ2opt[typ]}", f"{who}"] - opts += ["-s", f"{int(soft / 1024 ** 2)}m"] # round to MB - opts += ["-h", f"{int(hard / 1024 ** 2)}m"] # round to MB + opts += ["-s", f"{int(soft / 1024**2)}m"] # round to MB + opts += ["-h", f"{int(hard / 1024**2)}m"] # round to MB if inode_soft is not None: if inode_hard is None: @@ -1061,25 +1138,26 @@ def _set_quota(self, soft, who, obj=None, typ='user', hard=None, inode_soft=None elif inode_hard < inode_soft: self.log.raiseException( f"setQuota: can't set hard inode limit {inode_hard} lower then soft inode limit {inode_soft}", - GpfsOperationError) + GpfsOperationError, + ) opts += ["-S", str(inode_soft)] opts += ["-H", str(inode_hard)] opts.append(obj) - ec, _ = self._execute('tssetquota', opts, True) + ec, _ = self._execute("tssetquota", opts, True) if ec > 0: self.log.raiseException(f"_set_quota: tssetquota with opts {opts} failed", GpfsOperationError) def list_snapshots(self, filesystem): - """ List the snapshots of the given filesystem """ + """List the snapshots of the given filesystem""" try: - snaps = self._executeY('mmlssnapshot', [filesystem]) - return snaps['directory'] + snaps = self._executeY("mmlssnapshot", [filesystem]) + return snaps["directory"] except GpfsOperationError as err: - if 'No snapshots in file system' in err.args[0]: - self.log.debug('No snapshots in filesystem %s', filesystem) + if "No snapshots in file system" in err.args[0]: + self.log.debug("No snapshots in filesystem %s", filesystem) return [] else: self.log.raiseException(err.args[0], GpfsOperationError) @@ -1101,18 +1179,18 @@ def create_filesystem_snapshot(self, fsname, snapname, filesets=None): opts = [fsname, snapname] if filesets is not None: all_filesets = self.list_filesets(devices=fsname) - all_filesets_names = [x['filesetName'] for x in all_filesets[fsname].values()] + all_filesets_names = [x["filesetName"] for x in all_filesets[fsname].values()] if not all([x in all_filesets_names for x in filesets]): self.log.error("Not all given filesets could be found on filesystem %s!", fsname) return 0 - opts.extend(['-j', ','.join(filesets)]) + opts.extend(["-j", ",".join(filesets)]) - ec, out = self._execute('mmcrsnapshot', opts, True) + ec, out = self._execute("mmcrsnapshot", opts, True) if ec > 0: self.log.raiseException( - f"create_filesystem_snapshot: mmcrsnapshot with opts {opts} failed: {out}", - GpfsOperationError) + f"create_filesystem_snapshot: mmcrsnapshot with opts {opts} failed: {out}", GpfsOperationError + ) return ec == 0 @@ -1129,25 +1207,24 @@ def delete_filesystem_snapshot(self, fsname, snapname): return 0 opts = [fsname, snapname] - ec, out = self._execute('mmdelsnapshot', opts, True) + ec, out = self._execute("mmdelsnapshot", opts, True) if ec > 0: self.log.raiseException( - f"delete_filesystem_snapshot: mmdelsnapshot with opts {opts} failed: {out}", - GpfsOperationError) + f"delete_filesystem_snapshot: mmdelsnapshot with opts {opts} failed: {out}", GpfsOperationError + ) return ec == 0 def get_mmhealth_state(self): - """ Get the mmhealth state info of the GPFS components """ - opts = ['node', 'show'] - res = self._executeY('mmhealth', opts) - states = res['State'] - comp_entities = [f'{ident}_{value}' for ident, value in zip(states['component'], states['entityname'])] - return dict(zip(comp_entities, states['status'])) - + """Get the mmhealth state info of the GPFS components""" + opts = ["node", "show"] + res = self._executeY("mmhealth", opts) + states = res["State"] + comp_entities = [f"{ident}_{value}" for ident, value in zip(states["component"], states["entityname"])] + return dict(zip(comp_entities, states["status"])) def get_mmhealh_issues(self): - """ Get the mmhealth unhealthy events of the GPFS components """ - opts = ['node', 'show', '--unhealthy'] - res = self._executeY('mmhealth', opts) - events = res['Event'] - return zip(events['event'], events['message'], events['severity']) + """Get the mmhealth unhealthy events of the GPFS components""" + opts = ["node", "show", "--unhealthy"] + res = self._executeY("mmhealth", opts) + events = res["Event"] + return zip(events["event"], events["message"], events["severity"]) diff --git a/lib/vsc/filesystem/lustre.py b/lib/vsc/filesystem/lustre.py index b3cddd7..6236afb 100644 --- a/lib/vsc/filesystem/lustre.py +++ b/lib/vsc/filesystem/lustre.py @@ -17,6 +17,7 @@ @author: Kenneth Waegeman (Ghent University) """ + import os import re import glob @@ -31,45 +32,62 @@ import yaml from vsc.config.base import SCRATCH_SUBDIR, PROJECTS_SUBDIR, USERS_SUBDIR -LustreQuota = namedtuple('LustreQuota', - ['name', - 'blockUsage', 'blockQuota', 'blockLimit', 'blockGrace', 'blockInDoubt', - 'filesUsage', 'filesQuota', 'filesLimit', 'filesGrace', 'filesInDoubt', - 'filesetname']) +LustreQuota = namedtuple( + "LustreQuota", + [ + "name", + "blockUsage", + "blockQuota", + "blockLimit", + "blockGrace", + "blockInDoubt", + "filesUsage", + "filesQuota", + "filesLimit", + "filesGrace", + "filesInDoubt", + "filesetname", + ], +) # blockInDoubt and filesInDoubt does not exist in Lustre, so set to 0 # filesetname only valid for project quota LustreQuota.__new__.__defaults__ = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None) + class Typ2Opt(Enum): - user = 'u' - group = 'g' - project = 'p' + user = "u" + group = "g" + project = "p" + class Typ2Param(Enum): - USR = 'usr' - GRP = 'grp' - FILESET = 'prj' + USR = "usr" + GRP = "grp" + FILESET = "prj" + class Quotyp2Param(Enum): - block = 'dt' - inode = 'md' + block = "dt" + inode = "md" + class LustreOperationError(PosixOperationError): - """ Lustre Error """ + """Lustre Error""" + class LustreVscFSError(Exception): - """ LustreVSCFS Error """ + """LustreVSCFS Error""" + class LustreVscFS: """Default class for a vsc managed Lustre file system - Since Lustre doesn't have a 'lsfileset' kind of command, - we need some hints to defining names, ids and mappings - This can be a regular mapping function/hash,saved in some special file on lustre,.. - Alternatively we could replace this by changing the API + Since Lustre doesn't have a 'lsfileset' kind of command, + we need some hints to defining names, ids and mappings + This can be a regular mapping function/hash,saved in some special file on lustre,.. + Alternatively we could replace this by changing the API """ def __init__(self, mountpoint, project_locations, projectid_maps): - self.log = fancylogger.getLogger(name=self.__class__.__name__, fname=False) self.mountpoint = mountpoint self.project_locations = project_locations @@ -77,7 +95,7 @@ def __init__(self, mountpoint, project_locations, projectid_maps): self.pjparser = re.compile("([a-zA-Z]+)([0-9]+)") def pjid_from_name(self, name): - """ This only generates an id based on name and should be sanity_checked before using """ + """This only generates an id based on name and should be sanity_checked before using""" prefix, pjid = self.pjparser.match(name).groups() if prefix in self.projectid_maps.keys(): res = self.projectid_maps[prefix] + int(pjid) @@ -87,7 +105,7 @@ def pjid_from_name(self, name): return None def get_search_paths(self): - """ Get all the paths we should look for projects """ + """Get all the paths we should look for projects""" res = [] for loc in self.project_locations: res.extend(glob.glob(os.path.join(self.mountpoint, loc))) @@ -95,61 +113,58 @@ def get_search_paths(self): class LustreVscGhentScratchFs(LustreVscFS): - """ Make some assumptions on where to find filesets - This could also be extended to be done by importing config files """ + """Make some assumptions on where to find filesets + This could also be extended to be done by importing config files""" def __init__(self, mountpoint): - - project_locations = ['gent', 'gent/vo/00[0-9]'] - projectid_maps = {'gvo' : 900000} + project_locations = ["gent", "gent/vo/00[0-9]"] + projectid_maps = {"gvo": 900000} super().__init__(mountpoint, project_locations, projectid_maps) + class LustreVscScratchFs(LustreVscFS): def __init__(self, mountpoint): project_locations = [os.path.join(SCRATCH_SUBDIR, PROJECTS_SUBDIR), os.path.join(SCRATCH_SUBDIR, USERS_SUBDIR)] projectid_maps = {} - super().__init__(mountpoint, project_locations, - projectid_maps) + super().__init__(mountpoint, project_locations, projectid_maps) def pjid_from_name(self, name): del name - self.log.error('Use explicit mapping') - raise ValueError(f'Can not use pjid_from_name') + self.log.error("Use explicit mapping") + raise ValueError("Can not use pjid_from_name") + class LustreVscTier1cScratchFs(LustreVscFS): - """ Make some assumptions on where to find filesets - This could also be extended to be done by importing config files """ + """Make some assumptions on where to find filesets + This could also be extended to be done by importing config files""" def __init__(self, mountpoint): - - project_locations = ['gent', 'gent/projects/00[0-9]'] - projectid_maps = {'pj' : 900000} + project_locations = ["gent", "gent/projects/00[0-9]"] + projectid_maps = {"pj": 900000} super().__init__(mountpoint, project_locations, projectid_maps) - class LustreOperations(PosixOperations, metaclass=Singleton): - """ Lustre Operations """ + """Lustre Operations""" def __init__(self): super().__init__() - self.supportedfilesystems = ['lustre'] + self.supportedfilesystems = ["lustre"] self.filesystems = {} self.filesets = {} - self.quotadump = '/var/cache/lustre' + self.quotadump = "/var/cache/lustre" self.quota_types = Typ2Param self.default_mapping = LustreVscScratchFs def set_default_mapping(self, default_mapping=None): - ''' Set the class for the mapping of ids and search paths''' + """Set the class for the mapping of ids and search paths""" self.default_mapping = default_mapping def _execute_lfs(self, name, opts=None, changes=False): - """Return and check the LUSTRE lfs command. - """ + """Return and check the LUSTRE lfs command.""" - cmd = ['/usr/bin/lfs', name] + cmd = ["/usr/bin/lfs", name] cmd += opts ec, out = self._execute(cmd, changes) @@ -159,7 +174,7 @@ def _execute_lfs(self, name, opts=None, changes=False): return out def _execute_lctl_get_param_qmt_yaml(self, device, typ, quotyp=Quotyp2Param.block, qmt_direct=True): - """ executy LUSTRE lctl get_param qmt.* command and parse output + """executy LUSTRE lctl get_param qmt.* command and parse output eg: lctl get_param qmt.kwlust-*.dt*.glb-prj @@ -176,12 +191,12 @@ def _execute_lctl_get_param_qmt_yaml(self, device, typ, quotyp=Quotyp2Param.bloc """ - param = f'qmt.{device}-*.{quotyp.value}-*.glb-{typ.value}' + param = f"qmt.{device}-*.{quotyp.value}-*.glb-{typ.value}" if qmt_direct: - opts = ['get_param', param] + opts = ["get_param", param] res = self._execute_lctl(opts) else: - cmd = ['cat', os.path.join(self.quotadump, param)] + cmd = ["cat", os.path.join(self.quotadump, param)] ec, res = RunAsyncLoop.run(cmd) if ec != 0: self.log.raiseException(f"Could not get quota info. out:{res}", LustreOperationError) @@ -191,15 +206,15 @@ def _execute_lctl_get_param_qmt_yaml(self, device, typ, quotyp=Quotyp2Param.bloc newres = yaml.safe_load(quota_info[2]) except yaml.YAMLError as exc: self.log.raiseException( - f"_execute_lctl_get_param_qmt_yaml: Error in yaml output: {exc}", - LustreOperationError) + f"_execute_lctl_get_param_qmt_yaml: Error in yaml output: {exc}", LustreOperationError + ) return newres def _execute_lctl(self, opts, changes=False): - """ Return output of lctl command """ + """Return output of lctl command""" - cmd = ['/usr/sbin/lctl'] + cmd = ["/usr/sbin/lctl"] cmd += opts ec, out = self._execute(cmd, changes) @@ -209,7 +224,7 @@ def _execute_lctl(self, opts, changes=False): return out def list_filesystems(self, device=None): - """ List all Lustre file systems """ + """List all Lustre file systems""" if not self.localfilesystems: self._local_filesystems() @@ -220,32 +235,30 @@ def list_filesystems(self, device=None): lustrefss = {} for fs in self.localfilesystems: - if fs[0] == 'lustre': - fsloc = fs[3].split(':/') + if fs[0] == "lustre": + fsloc = fs[3].split(":/") if len(fsloc) == 2: fsname = fsloc[1] if not devices or fsname in devices: # keeping gpfs terminology - lustrefss[fsname] = { - 'defaultMountPoint': fs[1], - 'location': fsloc[0] - } + lustrefss[fsname] = {"defaultMountPoint": fs[1], "location": fsloc[0]} if not devices and not lustrefss: self.log.raiseException("No Lustre Filesystems found", LustreOperationError) elif not all(elem in lustrefss for elem in devices): - self.log.raiseException(f"Not all Lustre Filesystems of {devices} found, found {lustrefss}", - LustreOperationError) + self.log.raiseException( + f"Not all Lustre Filesystems of {devices} found, found {lustrefss}", LustreOperationError + ) return lustrefss def _get_fsinfo_for_path(self, path): - """ Get the Lustre file system name and mountpoint """ + """Get the Lustre file system name and mountpoint""" fs = self.what_filesystem(path) - return (fs[3].split(':/')[1], fs[1]) + return (fs[3].split(":/")[1], fs[1]) def _get_fshint_for_path(self, path): - """ Get hints to find projects locations and ids """ + """Get hints to find projects locations and ids""" fsname, fsmount = self._get_fsinfo_for_path(path) if fsname not in self.filesystems: if self.default_mapping is None: @@ -254,18 +267,18 @@ def _get_fshint_for_path(self, path): return self.filesystems[fsname] def _map_project_id(self, project_path, fileset_name): - """ Map a project name to a project id """ + """Map a project name to a project id""" fs = self._get_fshint_for_path(project_path) return fs.pjid_from_name(fileset_name) def _set_new_project_id(self, project_path, pjid): - """ Set the project id and flags for a new project path """ + """Set the project id and flags for a new project path""" exid = self.get_project_id(project_path, False) if not exid or int(exid) == 0: # recursive and inheritance flag set - opts = ['-p', pjid, '-r', '-s', project_path] - self._execute_lfs('project', opts, True) + opts = ["-p", pjid, "-r", "-s", project_path] + self._execute_lfs("project", opts, True) return pjid else: self.log.raiseException(f"Path {project_path} already has a projectid {exid}", LustreOperationError) @@ -273,53 +286,52 @@ def _set_new_project_id(self, project_path, pjid): return None def get_project_id(self, project_path, existing=True): - """ Parse lfs project output to get the project id for fileset """ + """Parse lfs project output to get the project id for fileset""" project_path = self._sanity_check(project_path) - opts = ['-d', project_path] + opts = ["-d", project_path] - res = self._execute_lfs('project', opts) + res = self._execute_lfs("project", opts) pjid, flag, path = res.split() - self.log.info('got pjid %s, flag %s, path %s', pjid, flag, path) - if flag == 'P' and path == project_path: + self.log.info("got pjid %s, flag %s, path %s", pjid, flag, path) + if flag == "P" and path == project_path: return pjid elif existing: self.log.raiseException( - f"Something went wrong fetching project id for {project_path}. Output was {res}", - LustreOperationError) + f"Something went wrong fetching project id for {project_path}. Output was {res}", LustreOperationError + ) else: - self.log.debug('path has no pjid set') + self.log.debug("path has no pjid set") return None def _quota_src(self, device): - """ Locate the quota info: directly through qmt params(True) or using a dump(False) """ + """Locate the quota info: directly through qmt params(True) or using a dump(False)""" - qparam = f'qmt.{device}-*.*.glb-*' - opts = ['list_param', qparam] + qparam = f"qmt.{device}-*.*.glb-*" + opts = ["list_param", qparam] try: self._execute_lctl(opts) except LustreOperationError: - cmd = ['ls', os.path.join(self.quotadump, qparam)] + cmd = ["ls", os.path.join(self.quotadump, qparam)] ec, _out = RunNoWorries.run(cmd) if ec != 0: self.log.raiseException("Could not get quota information from qmt or dump", LustreOperationError) return False else: - self.log.info('Getting quota information out of dump') + self.log.info("Getting quota information out of dump") return False else: - self.log.info('Running on Lustre Quota Target') + self.log.info("Running on Lustre Quota Target") return True - # pylint: disable=arguments-differ def list_quota(self, devices=None): """get quota info for filesystems for all user,group,project - Output has been remapped to format of gpfs.py - dict: key = deviceName, value is - dict with key quotaType (user | group | fileset) value is dict with - key = id, value dict with - key = remaining header entries and corresponding values as + Output has been remapped to format of gpfs.py + dict: key = deviceName, value is + dict with key quotaType (user | group | fileset) value is dict with + key = id, value dict with + key = remaining header entries and corresponding values as """ if devices is None: @@ -335,42 +347,42 @@ def list_quota(self, devices=None): blockres = self._execute_lctl_get_param_qmt_yaml(fsname, typp, Quotyp2Param.block, qmt_direct) inoderes = self._execute_lctl_get_param_qmt_yaml(fsname, typp, Quotyp2Param.inode, qmt_direct) for qentry in blockres: - qid = str(qentry['id']) - qlim = qentry['limits'] + qid = str(qentry["id"]) + qlim = qentry["limits"] # map quota fields to same names as for GPFS qinfo = { - 'name': qid, - 'blockUsage' : qlim['granted'], - 'blockQuota' : qlim['soft'], - 'blockLimit' : qlim['hard'], - 'blockGrace' : qlim['time'], + "name": qid, + "blockUsage": qlim["granted"], + "blockQuota": qlim["soft"], + "blockLimit": qlim["hard"], + "blockGrace": qlim["time"], } quota[fsname][typ][qid] = qinfo for qentry in inoderes: - qid = str(qentry['id']) - qlim = qentry['limits'] + qid = str(qentry["id"]) + qlim = qentry["limits"] quota[fsname][typ][qid].update({ - 'filesUsage' : qlim['granted'], - 'filesQuota' : qlim['soft'], - 'filesLimit' : qlim['hard'], - 'filesGrace' : qlim['time'], + "filesUsage": qlim["granted"], + "filesQuota": qlim["soft"], + "filesLimit": qlim["hard"], + "filesGrace": qlim["time"], }) for qid in quota[fsname][typ]: if typp == Typ2Param.FILESET: - quota[fsname][typ][qid]['filesetname'] = qid + quota[fsname][typ][qid]["filesetname"] = qid quota[fsname][typ][qid] = [LustreQuota(**quota[fsname][typ][qid])] return quota def _list_filesets(self, device): - """ Get all filesets for a Lustre device""" + """Get all filesets for a Lustre device""" - fs = self._get_fshint_for_path(device['defaultMountPoint']) + fs = self._get_fshint_for_path(device["defaultMountPoint"]) filesets = {} for upath in fs.get_search_paths(): spath = self._sanity_check(upath) - res = self._execute_lfs('project', [spath]) + res = self._execute_lfs("project", [spath]) for pjline in res.splitlines(): pjid, flag, path = pjline.split() @@ -381,27 +393,28 @@ def _list_filesets(self, device): if pjid in filesets: self.log.raiseException( f"projectids mapping multiple paths: {pjid}: {filesets[pjid]['path']}, {path}", - LustreOperationError) - elif flag != 'P': + LustreOperationError, + ) + elif flag != "P": # Not sure if this should give error or raise Exception self.log.raiseException( - f"Project inheritance flag not set for project {pjid}: {path}", - LustreOperationError) + f"Project inheritance flag not set for project {pjid}: {path}", LustreOperationError + ) else: path = self._sanity_check(path) - filesets[pjid] = {'path': path, 'filesetName': os.path.basename(path)} + filesets[pjid] = {"path": path, "filesetName": os.path.basename(path)} return filesets def set_fs_update(self, device): - """ Update this FS next run of list_filesets """ + """Update this FS next run of list_filesets""" del self.filesets[device] def get_fileset_info(self, filesystem_name, fileset_name): - """ get the info of a specific fileset """ + """get the info of a specific fileset""" fsets = self.list_filesets([filesystem_name]) for fileset in fsets[filesystem_name].values(): - if fileset['filesetName'] == fileset_name: + if fileset["filesetName"] == fileset_name: return fileset return None @@ -427,9 +440,9 @@ def list_filesets(self, devices=None): return filesetsres - - def make_fileset(self, new_fileset_path, fileset_name, inodes_max=1048576, fileset_id=None, - parent_fileset_name=None): + def make_fileset( + self, new_fileset_path, fileset_name, inodes_max=1048576, fileset_id=None, parent_fileset_name=None + ): """ Given path, create a new directory and set file quota - check uniqueness @@ -448,56 +461,64 @@ def make_fileset(self, new_fileset_path, fileset_name, inodes_max=1048576, files elif fileset_name != os.path.basename(fsetpath): self.log.raiseException( f"fileset name {fileset_name} should be the directory name {os.path.basename(fsetpath)}.", - LustreOperationError) + LustreOperationError, + ) # does the path exist ? if self.exists(fsetpath): self.log.raiseException( f"makeFileset for new_fileset_path {new_fileset_path} returned sane fsetpath {fsetpath}, " "but it already exists.", - LustreOperationError) + LustreOperationError, + ) parentfsetpath = os.path.dirname(fsetpath) if not self.exists(parentfsetpath): self.log.raiseException( f"parent dir {parentfsetpath} of fsetpath {fsetpath} does not exist. Not going to create it.", - LustreOperationError) - + LustreOperationError, + ) fsname, _fsmount = self._get_fsinfo_for_path(parentfsetpath) fsinfo = self.get_fileset_info(fsname, fileset_name) if fsinfo: # bail if there is a fileset with the same name self.log.raiseException( - f"Found existing fileset {fileset_name} with the same name at {fsinfo['path']}", - LustreOperationError) + f"Found existing fileset {fileset_name} with the same name at {fsinfo['path']}", LustreOperationError + ) return None pjid = str(fileset_id) if fileset_id else self._map_project_id(parentfsetpath, fileset_name) filesets = self.list_filesets([fsname]) if pjid in filesets[fsname]: self.log.raiseException( - f"Found existing projectid {pjid} in file system {fsname}: {filesets[pjid]}", - LustreOperationError) + f"Found existing projectid {pjid} in file system {fsname}: {filesets[pjid]}", LustreOperationError + ) # create the fileset: dir and project self.make_dir(fsetpath) try: self._set_new_project_id(fsetpath, pjid) # set inode default quota; block quota should be set after with set_fileset_quota, default 1MB - blockq = 1024 ** 2 - self._set_quota(who=pjid, obj=fsetpath, typ=Typ2Opt.project, - soft=blockq, hard=blockq, inode_soft=inodes_max, inode_hard=inodes_max) + blockq = 1024**2 + self._set_quota( + who=pjid, + obj=fsetpath, + typ=Typ2Opt.project, + soft=blockq, + hard=blockq, + inode_soft=inodes_max, + inode_hard=inodes_max, + ) except LustreOperationError as err: self.log.error("Something went wrong creating fileset %s with id %s, error: %s", fsetpath, pjid, err) - os.rmdir(fsetpath) # only deletes empty directories + os.rmdir(fsetpath) # only deletes empty directories self.log.raiseException("Fileset creation failed, fileset directory removed", LustreOperationError) self.log.info("Created new fileset %s at %s with id %s", fileset_name, fsetpath, pjid) self.set_fs_update(fsname) return True - def set_user_quota(self, soft, user, obj=None, hard=None, inode_soft=None, inode_hard=None): """Set quota for a user. @@ -508,8 +529,9 @@ def set_user_quota(self, soft, user, obj=None, hard=None, inode_soft=None, inode @type inode_soft: integer representing the soft files limit @type inode_soft: integer representing the hard files quota """ - self._set_quota(who=user, obj=obj, typ=Typ2Opt.user, soft=soft, hard=hard, - inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota( + who=user, obj=obj, typ=Typ2Opt.user, soft=soft, hard=hard, inode_soft=inode_soft, inode_hard=inode_hard + ) def set_group_quota(self, soft, group, obj=None, hard=None, inode_soft=None, inode_hard=None): """Set quota for a group on a given object (e.g., a path in the filesystem, which may correpond to a fileset) @@ -521,8 +543,9 @@ def set_group_quota(self, soft, group, obj=None, hard=None, inode_soft=None, ino @type inode_soft: integer representing the soft files limit @type inode_soft: integer representing the hard files quota """ - self._set_quota(who=group, obj=obj, typ=Typ2Opt.group, soft=soft, hard=hard, - inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota( + who=group, obj=obj, typ=Typ2Opt.group, soft=soft, hard=hard, inode_soft=inode_soft, inode_hard=inode_hard + ) def set_fileset_quota(self, soft, fileset_path, fileset_name=None, hard=None, inode_soft=None, inode_hard=None): """Set quota on a fileset. This maps to projects in Lustre @@ -537,16 +560,24 @@ def set_fileset_quota(self, soft, fileset_path, fileset_name=None, hard=None, in fileset_path = self._sanity_check(fileset_path) if fileset_name is not None and fileset_name != os.path.basename(fileset_path): self.log.raiseException( - f'fileset name {fileset_name} should be the directory name {os.path.basename(fileset_path)}.', - LustreOperationError) + f"fileset name {fileset_name} should be the directory name {os.path.basename(fileset_path)}.", + LustreOperationError, + ) # we need the corresponding project id project = self.get_project_id(fileset_path) if int(project) == 0: self.log.raiseException("Can not set quota for fileset with projectid 0", LustreOperationError) else: - self._set_quota(who=project, obj=fileset_path, typ=Typ2Opt.project, soft=soft, hard=hard, - inode_soft=inode_soft, inode_hard=inode_hard) + self._set_quota( + who=project, + obj=fileset_path, + typ=Typ2Opt.project, + soft=soft, + hard=hard, + inode_soft=inode_soft, + inode_hard=inode_hard, + ) def set_user_grace(self, obj, grace=0, who=None): """Set the grace period for user data. @@ -586,14 +617,14 @@ def _set_grace(self, obj, typ, grace=0): if not self.dry_run and not self.exists(obj): self.log.raiseException(f"setQuota: can't set quota on none-existing obj {obj}", LustreOperationError) - opts = ['-t'] + opts = ["-t"] opts += [f"-{typ.value}"] opts += ["-b", f"{int(grace)}"] opts += ["-i", f"{int(grace)}"] opts.append(obj) - self._execute_lfs('setquota', opts, True) + self._execute_lfs("setquota", opts, True) def _get_quota(self, who, obj, typ, human=False): """Get quota of a given object. @@ -613,11 +644,11 @@ def _get_quota(self, who, obj, typ, human=False): opts.append("-h") opts.append(obj) - res = self._execute_lfs('quota', opts) + res = self._execute_lfs("quota", opts) return res def get_project_quota(self, who, obj, human=True): - """ Return project quota""" + """Return project quota""" return self._get_quota(who, obj, Typ2Opt.project, human) def _set_quota(self, who, obj, typ=Typ2Opt.user, soft=None, hard=None, inode_soft=None, inode_hard=None): @@ -642,18 +673,19 @@ def _set_quota(self, who, obj, typ=Typ2Opt.user, soft=None, hard=None, inode_sof opts = [f"-{typ.value}", f"{who}"] if soft is None and inode_soft is None: - self.log.raiseException("setQuota: At least one type of quota (block,inode) should be specified", - LustreOperationError) + self.log.raiseException( + "setQuota: At least one type of quota (block,inode) should be specified", LustreOperationError + ) if soft: if hard is None: hard = int(soft * soft2hard_factor) elif hard < soft: self.log.raiseException( - f"setQuota: can't set hard limit {hard} lower then soft limit {soft}", - LustreOperationError) - softm = int(soft / 1024 ** 2) # round to MB - hardm = int(hard / 1024 ** 2) # round to MB + f"setQuota: can't set hard limit {hard} lower then soft limit {soft}", LustreOperationError + ) + softm = int(soft / 1024**2) # round to MB + hardm = int(hard / 1024**2) # round to MB if softm == 0 or hardm == 0: self.log.raiseException("setQuota: setting quota to 0 would be infinite quota", LustreOperationError) else: @@ -666,11 +698,12 @@ def _set_quota(self, who, obj, typ=Typ2Opt.user, soft=None, hard=None, inode_sof elif inode_hard < inode_soft: self.log.raiseException( f"setQuota: can't set hard inode limit {inode_hard} lower then soft inode limit {inode_soft}", - LustreOperationError) + LustreOperationError, + ) opts += ["-i", str(inode_soft)] opts += ["-I", str(inode_hard)] opts.append(obj) - self._execute_lfs('setquota', opts, True) + self._execute_lfs("setquota", opts, True) diff --git a/lib/vsc/filesystem/operator.py b/lib/vsc/filesystem/operator.py index 47f3407..39626eb 100644 --- a/lib/vsc/filesystem/operator.py +++ b/lib/vsc/filesystem/operator.py @@ -17,12 +17,14 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ + import importlib import logging -STORAGE_OPERATORS = ('Posix', 'Gpfs', 'OceanStor', 'Lustre') -OPERATOR_CLASS_SUFFIX = 'Operations' -OPERATOR_ERROR_CLASS_SUFFIX = 'OperationError' +STORAGE_OPERATORS = ("Posix", "Gpfs", "OceanStor", "Lustre") +OPERATOR_CLASS_SUFFIX = "Operations" +OPERATOR_ERROR_CLASS_SUFFIX = "OperationError" + class StorageOperator: """ @@ -66,7 +68,7 @@ def import_operator(backend): except NameError: ModuleImportError = ImportError - backend_module_name = '.'.join(['vsc', 'filesystem', backend]) + backend_module_name = ".".join(["vsc", "filesystem", backend]) try: backend_module = importlib.import_module(backend_module_name) except ModuleImportError: diff --git a/lib/vsc/filesystem/posix.py b/lib/vsc/filesystem/posix.py index aa8547c..c680b11 100644 --- a/lib/vsc/filesystem/posix.py +++ b/lib/vsc/filesystem/posix.py @@ -18,7 +18,6 @@ @author: Stijn De Weirdt (Ghent University) """ - import errno import os import stat @@ -27,22 +26,22 @@ from vsc.utils.patterns import Singleton from vsc.utils.run import asyncloop -OS_LINUX_MOUNTS = '/proc/mounts' -OS_LINUX_FILESYSTEMS = '/proc/filesystems' +OS_LINUX_MOUNTS = "/proc/mounts" +OS_LINUX_FILESYSTEMS = "/proc/filesystems" # be very careful to add new ones here OS_LINUX_IGNORE_FILESYSTEMS = ( - 'rootfs', # special initramfs filesystem - 'configfs', # kernel config - 'debugfs', # kernel debug - 'tracefs', # kernel trace - 'usbfs', # usb devices - 'ipathfs', # qlogic IB - 'binfmt_misc', # ? - 'rpc_pipefs', # NFS RPC - 'fuse.sshfs', # X2GO sshfs over fuse - 'fuse.irods', # irods fuse - 'fuse.irodsfs', # irods nfs fuse + "rootfs", # special initramfs filesystem + "configfs", # kernel config + "debugfs", # kernel debug + "tracefs", # kernel trace + "usbfs", # usb devices + "ipathfs", # qlogic IB + "binfmt_misc", # ? + "rpc_pipefs", # NFS RPC + "fuse.sshfs", # X2GO sshfs over fuse + "fuse.irods", # irods fuse + "fuse.irodsfs", # irods nfs fuse ) @@ -89,7 +88,7 @@ def _execute(self, cmd, changes=False): def _sanity_check(self, obj=None, force_ignorerealpath=False): """Run sanity check on obj. E.g. force absolute path. - @type obj: string to check + @type obj: string to check """ # filler for reducing lots of LOC if obj is None: @@ -99,12 +98,13 @@ def _sanity_check(self, obj=None, force_ignorerealpath=False): else: obj = self.obj - obj = obj.rstrip('/') # remove trailing / + obj = obj.rstrip("/") # remove trailing / if self.forceabsolutepath: if not os.path.isabs(obj): # other test: obj.startswith(os.path.sep) - self.log.raiseException(f"_sanity_check check absolute path: obj {obj} is not an absolute path", - PosixOperationError) + self.log.raiseException( + f"_sanity_check check absolute path: obj {obj} is not an absolute path", PosixOperationError + ) return None # check if filesystem matches current class @@ -123,7 +123,8 @@ def _sanity_check(self, obj=None, force_ignorerealpath=False): self.log.raiseException( f"_sanity_check found filesystem {tmpfs[0]} for subpath {fp} of obj {obj} is not a " "supported filesystem (supported {self.supportedfilesystems})", - PosixOperationError) + PosixOperationError, + ) if filesystem is None: self.log.raiseException(f"_sanity_check no valid filesystem found for obj {obj}", PosixOperationError) @@ -136,19 +137,19 @@ def _sanity_check(self, obj=None, force_ignorerealpath=False): if not obj == os.path.realpath(obj): # some part of the path is a symlink if ignore_real_path_mismatch: - self.log.debug("_sanity_check obj %s doesn't correspond with realpath %s", - obj, os.path.realpath(obj)) + self.log.debug("_sanity_check obj %s doesn't correspond with realpath %s", obj, os.path.realpath(obj)) else: self.log.raiseException( f"_sanity_check obj {obj} doesn't correspond with realpath {os.path.realpath(obj)}", - PosixOperationError) + PosixOperationError, + ) return None return obj def set_object(self, obj): """Set the object, apply checks if needed - @type obj: string to set as obj + @type obj: string to set as obj """ self.obj = self._sanity_check(obj) @@ -159,7 +160,7 @@ def exists(self, obj=None): def _exists(self, obj): """Based on obj, check if obj exists or not - called by _sanity_check and exists or with sanitised obj + called by _sanity_check and exists or with sanitised obj """ if obj is None: self.log.raiseException("_exists: obj is None", PosixOperationError) @@ -210,17 +211,19 @@ def _what_filesystem(self, obj): self.log.exception("Failed to get fsid from obj %s", obj) return None - fss = [x for x in self.localfilesystems if x[self.localfilesystemnaming.index('id')] == fsid] + fss = [x for x in self.localfilesystems if x[self.localfilesystemnaming.index("id")] == fsid] if len(fss) == 0: self.log.raiseException( f"No matching filesystem found for obj {obj} id {fsid} (localfilesystems: {self.localfilesystems})", - PosixOperationError) + PosixOperationError, + ) elif len(fss) > 1: self.log.raiseException( f"More than one matching filesystem found for obj {obj} with id {fsid} " f"(matched localfilesystems: {fsid})", - PosixOperationError) + PosixOperationError, + ) else: self.log.debug("Found filesystem for obj %s: %s", obj, fss[0]) return fss[0] @@ -232,18 +235,19 @@ def _local_filesystems(self): if not os.path.isfile(OS_LINUX_MOUNTS): self.log.raiseException(f"Missing Linux OS overview of mounts {OS_LINUX_MOUNTS}", PosixOperationError) if not os.path.isfile(OS_LINUX_FILESYSTEMS): - self.log.raiseException(f"Missing Linux OS overview of filesystems {OS_LINUX_FILESYSTEMS}", - PosixOperationError) + self.log.raiseException( + f"Missing Linux OS overview of filesystems {OS_LINUX_FILESYSTEMS}", PosixOperationError + ) try: with open(OS_LINUX_MOUNTS) as mounts: currentmounts = [x.strip().split(" ") for x in mounts.readlines()] # returns [('rootfs', '/', 2051L, 'rootfs'), ('ext4', '/', 2051L, '/dev/root'), # ('tmpfs', '/dev', 17L, '/dev'), ... - self.localfilesystemnaming = ['type', 'mountpoint', 'id', 'device'] + self.localfilesystemnaming = ["type", "mountpoint", "id", "device"] # do we need further parsing, eg of autofs types or remove pseudo filesystems ? if self.ignorefilesystems: - currentmounts = [x for x in currentmounts if not x[2] in OS_LINUX_IGNORE_FILESYSTEMS] + currentmounts = [x for x in currentmounts if x[2] not in OS_LINUX_IGNORE_FILESYSTEMS] self.localfilesystems = [[y[2], y[1], os.stat(y[1]).st_dev, y[0]] for y in currentmounts] except OSError: self.log.exception("Failed to create the list of current mounted filesystems") @@ -251,7 +255,7 @@ def _local_filesystems(self): def _largest_existing_path(self, obj): """Given obj /a/b/c/d, check which subpath exists and will determine eg filesystem type of obj. - Start with /a/b/c/d, then /a/b/c etc + Start with /a/b/c/d, then /a/b/c etc """ obj = self._sanity_check(obj) @@ -293,8 +297,7 @@ def make_symlink(self, target, obj=None, force=True): os.unlink(target) target = self._sanity_check(target_) elif not self.dry_run: - self.log.raiseException(f"Target {target} does not exist, cannot make symlink to it", - PosixOperationError) + self.log.raiseException(f"Target {target} does not exist, cannot make symlink to it", PosixOperationError) self.log.info("Attempting to create a symlink from %s to %s", obj, target) if self.exists(obj): @@ -305,8 +308,9 @@ def make_symlink(self, target, obj=None, force=True): else: os.unlink(obj) except OSError: - self.log.raiseException(f"Cannot unlink existing symlink from {obj} to {target}", - PosixOperationError) + self.log.raiseException( + f"Cannot unlink existing symlink from {obj} to {target}", PosixOperationError + ) else: self.log.info("Symlink already exists from %s to %s", obj, target) return # Nothing to do, symlink already exists @@ -367,19 +371,19 @@ def populate_home_dir(self, user_id, group_id, home_dir, ssh_public_keys): """ # ssh self.log.info("Populating home %s for user %s:%s", home_dir, user_id, group_id) - ssh_path = os.path.join(home_dir, '.ssh') + ssh_path = os.path.join(home_dir, ".ssh") self.make_dir(ssh_path) self.log.info("Placing %d ssh public keys in the authorized keys file.", len(ssh_public_keys)) - authorized_keys = os.path.join(home_dir, '.ssh', 'authorized_keys') + authorized_keys = os.path.join(home_dir, ".ssh", "authorized_keys") - default_keys = ['dsa', 'rsa', 'ed25519'] + default_keys = ["dsa", "rsa", "ed25519"] default_public_keys = [] if self.dry_run: self.log.info("Writing ssh keys. Dry-run, so not really doing anything.") else: for default_key in default_keys: - default_key_file = os.path.join(home_dir, '.ssh', f'id_{default_key}.pub') + default_key_file = os.path.join(home_dir, ".ssh", f"id_{default_key}.pub") if os.path.exists(default_key_file): with open(default_key_file) as fp: default_public_keys.append(fp.readline()) @@ -390,26 +394,26 @@ def populate_home_dir(self, user_id, group_id, home_dir, ssh_public_keys): else: self.log.info("No default key found, not adding to authorized_keys") - with open(authorized_keys, 'w') as fp: - fp.write("\n".join(ssh_public_keys + [''])) + with open(authorized_keys, "w") as fp: + fp.write("\n".join(ssh_public_keys + [""])) self.chmod(0o644, authorized_keys) self.chmod(0o700, ssh_path) # bash bashprofile_text = [ - 'if [ -f ~/.bashrc ]; then', - ' . ~/.bashrc', - 'fi', + "if [ -f ~/.bashrc ]; then", + " . ~/.bashrc", + "fi", ] bashrc_text = [ - '# do NOT remove the following lines:', - 'if [ -f /etc/bashrc ]; then', - ' . /etc/bashrc', - 'fi', + "# do NOT remove the following lines:", + "if [ -f /etc/bashrc ]; then", + " . /etc/bashrc", + "fi", ] - bashrc_path = os.path.join(home_dir, '.bashrc') - bashprofile_path = os.path.join(home_dir, '.bash_profile') + bashrc_path = os.path.join(home_dir, ".bashrc") + bashprofile_path = os.path.join(home_dir, ".bash_profile") if self.dry_run: self.log.info("Writing .bashrc an .bash_profile. Dry-run, so not really doing anything.") if not os.path.exists(bashprofile_path): @@ -419,17 +423,20 @@ def populate_home_dir(self, user_id, group_id, home_dir, ssh_public_keys): else: self._deploy_dot_file(bashrc_path, ".bashrc", user_id, bashrc_text) self._deploy_dot_file(bashprofile_path, ".bash_profile", user_id, bashprofile_text) - for f in [home_dir, - os.path.join(home_dir, '.ssh'), - os.path.join(home_dir, '.ssh', 'authorized_keys'), - os.path.join(home_dir, '.bashrc'), - os.path.join(home_dir, '.bash_profile')]: + for f in [ + home_dir, + os.path.join(home_dir, ".ssh"), + os.path.join(home_dir, ".ssh", "authorized_keys"), + os.path.join(home_dir, ".bashrc"), + os.path.join(home_dir, ".bash_profile"), + ]: self.log.info("Changing ownership of %s to %s:%s", f, user_id, group_id) try: self.chown(user_id, group_id, f, force_ignorerealpath=True) except OSError: self.log.raiseException( - f"Cannot change ownership of file {f} to {user_id}:{group_id}", PosixOperationError) + f"Cannot change ownership of file {f} to {user_id}:{group_id}", PosixOperationError + ) def _deploy_dot_file(self, path, filename, user_id, contents): """ @@ -445,19 +452,19 @@ def _deploy_dot_file(self, path, filename, user_id, contents): if os.path.islink(path): self.log.info("%s is symlinked to non-existing target %s ", filename, os.path.realpath(path)) os.unlink(path) - with open(path, 'w') as fp: - fp.write("\n".join(contents + [''])) + with open(path, "w") as fp: + fp.write("\n".join(contents + [""])) def list_quota(self, obj=None): """Report on quota""" obj = self._sanity_check(obj) self.log.error("listQuota not implemented for this class %s", self.__class__.__name__) - def set_quota(self, soft, who, obj=None, typ='user', hard=None, grace=None): + def set_quota(self, soft, who, obj=None, typ="user", hard=None, grace=None): """Set quota - @type soft: int, soft limit in bytes - @type who: identifier (eg username or userid) - @type grace: int, grace period in seconds + @type soft: int, soft limit in bytes + @type who: identifier (eg username or userid) + @type grace: int, grace period in seconds """ del grace del hard @@ -474,13 +481,13 @@ def chown(self, owner, group=None, obj=None, force_ignorerealpath=False): self.log.info("Changing ownership of %s to %s:%s", obj, owner, group) try: if self.dry_run: - self.log.info("Chown on %s to %s:%s. Dry-run, so not actually changing this ownership", - obj, owner, group) + self.log.info( + "Chown on %s to %s:%s. Dry-run, so not actually changing this ownership", obj, owner, group + ) else: os.chown(obj, owner, group) except OSError: - self.log.raiseException(f"Cannot change ownership of object {obj} to {owner}:{group}", - PosixOperationError) + self.log.raiseException(f"Cannot change ownership of object {obj} to {owner}:{group}", PosixOperationError) def chmod(self, permissions, obj=None): """Change permissions on the object. @@ -494,13 +501,15 @@ def chmod(self, permissions, obj=None): try: if self.dry_run: - self.log.info("Chmod on %s to %s. Dry-run, so not actually changing access permissions", - obj, permissions) + self.log.info( + "Chmod on %s to %s. Dry-run, so not actually changing access permissions", obj, permissions + ) else: os.chmod(obj, permissions) except OSError: - self.log.raiseException(f"Could not change the permissions on object {obj} to {permissions:o}", - PosixOperationError) + self.log.raiseException( + f"Could not change the permissions on object {obj} to {permissions:o}", PosixOperationError + ) def compare_files(self, target, obj=None): """Compare obj and target.""" diff --git a/ruff.toml b/ruff.toml index 3df3c78..2aad060 100644 --- a/ruff.toml +++ b/ruff.toml @@ -3,7 +3,7 @@ indent-width = 4 preview = true [lint] extend-select = ['E101', 'E501', 'E713', 'E4', 'E7', 'E9', 'F', 'F811', 'W291', 'PLR0911', 'PLW0602', 'PLW0604', 'PLW0108', 'PLW0127', 'PLW0129', 'PLW1501', 'PLR0124', 'PLR0202', 'PLR0203', 'PLR0402', 'PLR0913', 'B028', 'B905', 'C402', 'C403', 'UP032', 'UP037', 'UP025', 'UP036', 'UP034', 'UP033', 'UP031', 'UP004'] -exclude = ['.bzr', '.direnv', '.eggs', '.git', '.git-rewrite', '.hg', '.ipynb_checkpoints', '.mypy_cache', '.nox', '.pants.d', '.pyenv', '.pytest_cache', '.pytype', '.ruff_cache', '.svn', '.tox', '.venv', '.vscode', '__pypackages__', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'site-packages', 'venv', 'test/prospectortest/*'] +exclude = ['.bzr', '.direnv', '.eggs', '.git', '.git-rewrite', '.hg', '.ipynb_checkpoints', '.mypy_cache', '.nox', '.pants.d', '.pyenv', '.pytest_cache', '.pytype', '.ruff_cache', '.svn', '.tox', '.venv', '.vscode', '__pypackages__', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'site-packages', 'venv', 'test/*'] ignore = ['E731'] pylint.max-args = 11 [format] @@ -12,4 +12,4 @@ indent-style = "space" docstring-code-format = true docstring-code-line-length = 120 line-ending = "lf" -exclude = ["test/prospectortest/*"] +exclude = ["test/*"] diff --git a/setup.py b/setup.py index d1b4db6..da0d95a 100755 --- a/setup.py +++ b/setup.py @@ -18,9 +18,10 @@ @author: Stijn De Weirdt (Ghent University) @author: Andy Georges (Ghent University) """ + import sys -import vsc.install.shared_setup as shared_setup +from vsc.install import shared_setup from vsc.install.shared_setup import ag, kh, sdw, kw, wdp install_requires = [ @@ -29,13 +30,12 @@ "vsc-utils >= 2.0.0", ] -if sys.version_info < (3,9): +if sys.version_info < (3, 9): # noqa: UP036 install_requires.append("pyyaml <= 6.0.1") else: install_requires.append("pyyaml") - PACKAGE = { "version": "2.3.2", "author": [sdw, ag, kh, kw],