Change the main to rely on the Sync class (HPC-7469)

bin/sync_vsc_users.py

@@ -29,18 +29,12 @@
 """
 
 import logging
-import sys
 
-from vsc.accountpage.client import AccountpageClient
-from vsc.accountpage.wrappers import mkVscUserSizeQuota
+from vsc.accountpage.sync import Sync
 from vsc.administration.user import process_users, process_users_quota
 from vsc.administration.vo import process_vos
 from vsc.config.base import GENT
 from vsc.utils import fancylogger
-from vsc.utils.missing import nub
-from vsc.utils.nagios import NAGIOS_EXIT_CRITICAL
-from vsc.utils.script_tools import ExtendedSimpleOption
-from vsc.utils.timestamp import convert_timestamp, write_timestamp, retrieve_timestamp_with_default
 
 NAGIOS_HEADER = "sync_vsc_users"
 NAGIOS_CHECK_INTERVAL_THRESHOLD = 15 * 60  # 15 minutes
@@ -64,17 +58,9 @@ class UserGroupStatusUpdateError(Exception):
     pass
 
 
-def main():
-    """
-    Main script.
-    - build the filter
-    - fetches the users
-    - process the users
-    - write the new timestamp if everything went OK
-    - write the nagios check file
-    """
+class VscUserSync(Sync):
 
-    options = {
+    CLI_OPTIONS = {
         'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD,
         'storage': ('storage systems on which to deploy users and vos', None, 'extend', []),
         'user': ('process users', None, 'store_true', False),
@@ -85,97 +71,88 @@ def main():
         'start_timestamp': ('Timestamp to start the sync from', str, 'store', None),
     }
 
-    opts = ExtendedSimpleOption(options)
-    stats = {}
+    def do(self, dry_run):
+        """
+        Actual work
+        - build the filter
+        - fetches the users
+        - process the users
+        - write the new timestamp if everything went OK
+        - write the nagios check file
+        """
 
-    (last_timestamp, start_time) = retrieve_timestamp_with_default(
-        SYNC_TIMESTAMP_FILENAME,
-        start_timestamp=opts.options.start_timestamp)
-    logging.info("Using timestamp %s", last_timestamp)
-    logging.info("Using startime %s", start_time)
-
-    try:
-        client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/")
-
-        institute = opts.options.host_institute
+        stats = {}
+        institute = self.options.host_institute
 
         (users_ok, users_fail) = ([], [])
         (quota_ok, quota_fail) = ([], [])
-        if opts.options.user:
-            changed_accounts = client.account.institute[institute].modified[last_timestamp].get()[1]
+        if self.options.user:
+            changed_accounts, _ = self.apc.get_accounts()  # we ignore inactive accounts
 
             logging.info("Found %d %s accounts that have changed in the accountpage since %s" %
-                        (len(changed_accounts), institute, last_timestamp))
-
-            accounts = nub([u['vsc_id'] for u in changed_accounts])
+                        (len(changed_accounts), institute, self.start_timestamp))
 
-            for storage_name in opts.options.storage:
+            for storage_name in self.options.storage:
                 (users_ok, users_fail) = process_users(
-                    opts.options,
-                    accounts,
+                    changed_accounts,
                     storage_name,
-                    client,
-                    institute)
+                    self.apc,
+                    institute,
+                    self.options.dry_run)
                 stats["%s_users_sync" % (storage_name,)] = len(users_ok)
                 stats["%s_users_sync_fail" % (storage_name,)] = len(users_fail)
                 stats["%s_users_sync_fail_warning" % (storage_name,)] = STORAGE_USERS_LIMIT_WARNING
                 stats["%s_users_sync_fail_critical" % (storage_name,)] = STORAGE_USERS_LIMIT_CRITICAL
 
-            for storage_name in opts.options.storage:
-                storage_changed_quota = [mkVscUserSizeQuota(q) for q in
-                                         client.quota.user.storage[storage_name].modified[last_timestamp].get()[1]]
-                storage_changed_quota = [q for q in storage_changed_quota if q.fileset.startswith('vsc')]
-                logging.info("Found %d accounts that have changed quota on storage %s in the accountpage since %s",
-                            len(storage_changed_quota), storage_name, last_timestamp)
+            for storage_name in self.options.storage:
+                storage_changed_quota = [q for q in self.get_user_storage_quota(storage_name=storage_name)
+                    if q.fileset.startswith('vsc')]
+                logging.info("Found %d quota changes on storage %s in the accountpage",
+                            len(storage_changed_quota), storage_name)
                 (quota_ok, quota_fail) = process_users_quota(
-                    opts.options,
                     storage_changed_quota,
                     storage_name,
-                    client,
-                    institute)
+                    self.apc,
+                    institute,
+                    self.options.dry_run)
                 stats["%s_quota_sync" % (storage_name,)] = len(quota_ok)
                 stats["%s_quota_sync_fail" % (storage_name,)] = len(quota_fail)
                 stats["%s_quota_sync_fail_warning" % (storage_name,)] = STORAGE_QUOTA_LIMIT_WARNING
                 stats["%s_quota_sync_fail_critical" % (storage_name,)] = STORAGE_QUOTA_LIMIT_CRITICAL
 
         (vos_ok, vos_fail) = ([], [])
-        if opts.options.vo:
+        if self.options.vo:
             # FIXME: when api has changed, limit to modified per institute here
-            changed_vos = client.vo.modified[last_timestamp].get()[1]
-            changed_vo_quota = client.quota.vo.modified[last_timestamp].get()[1]
+            changed_groups, _ = self.apc.get_groups()  # we ignore inactive groups
+            changed_vos = [g for g in changed_groups if g.vsc_id.startswith("gvo") and not g.vsc_id.startswith("gvos")]
+            changed_vo_quota = [q for q in self.apc.get_vo_storage_quota(storage_name=storage_name)
+                if q.fileset.startswith('gvo') and not q.fileset.startswith('gvos')]
 
-            vos = sorted(set([v['vsc_id'] for v in changed_vos] +
-                             [v['virtual_organisation'] for v in changed_vo_quota]))
+            vos = sorted(set([v.vsc_id for v in changed_vos] + [v.virtual_organisation for v in changed_vo_quota]))
 
             logging.info("Found %d %s VOs that have changed in the accountpage since %s" %
-                        (len(changed_vos), institute, last_timestamp))
+                        (len(changed_vos), institute, self.start_timestamp))
             logging.info("Found %d %s VOs that have changed quota in the accountpage since %s" %
-                        (len(changed_vo_quota), institute, last_timestamp))
+                        (len(changed_vo_quota), institute, self.start_timestamp))
             logging.debug("Found the following {institute} VOs: {vos}".format(institute=institute, vos=vos))
 
-            for storage_name in opts.options.storage:
+            for storage_name in self.options.storage:
                 (vos_ok, vos_fail) = process_vos(
-                    opts.options,
                     vos,
                     storage_name,
-                    client,
-                    last_timestamp,
-                    institute)
+                    self.apc,
+                    self.start_timestamp,
+                    institute,
+                    self.options.dry_run)
                 stats["%s_vos_sync" % (storage_name,)] = len(vos_ok)
                 stats["%s_vos_sync_fail" % (storage_name,)] = len(vos_fail)
                 stats["%s_vos_sync_fail_warning" % (storage_name,)] = STORAGE_VO_LIMIT_WARNING
                 stats["%s_vos_sync_fail_critical" % (storage_name,)] = STORAGE_VO_LIMIT_CRITICAL
 
-        if not (users_fail or quota_fail or vos_fail) and not opts.options.dry_run:
-            (_, ldap_timestamp) = convert_timestamp(start_time)
-            write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
-    except Exception as err:
-        logger.exception("critical exception caught: %s" % (err))
-        opts.critical("Script failed in a horrible way")
-        sys.exit(NAGIOS_EXIT_CRITICAL)
-
-    opts.epilogue("%s users and VOs synchronised" % institute, stats)
-
+        if users_fail or quota_fail or vos_fail:
+            return users_fail + quota_fail + vos_fail
+        else:
+            return False
 
 if __name__ == '__main__':
-    main()
+    VscUserSync().main()

lib/vsc/administration/user.py

@@ -451,19 +451,20 @@ def update_user_status(user, client):
                                         (user.user_id, account.status))
 
 
-def process_users_quota(options, user_quota, storage_name, client, host_institute=GENT, use_user_cache=True):
+def process_users_quota(user_quota, storage_name, client, host_institute=GENT, use_user_cache=True, dry_run=False):
     """
     Process the users' quota for the given storage.
     """
     error_quota = []
     ok_quota = []
 
     for quota in user_quota:
-        user = VscTier2AccountpageUser(quota.user,
-                                       rest_client=client,
-                                       host_institute=host_institute,
-                                       use_user_cache=use_user_cache)
-        user.dry_run = options.dry_run
+        user = VscTier2AccountpageUser(
+            quota.user,
+            rest_client=client,
+            host_institute=host_institute,
+            use_user_cache=use_user_cache)
+        user.dry_run = dry_run
 
         try:
             if storage_name == VSC_HOME:
@@ -483,7 +484,7 @@ def process_users_quota(options, user_quota, storage_name, client, host_institut
     return (ok_quota, error_quota)
 
 
-def process_users(options, account_ids, storage_name, client, host_institute=GENT, use_user_cache=True):
+def process_users(account_ids, storage_name, client, host_institute=GENT, use_user_cache=True, dry_run=False):
     """
     Process the users.
 
@@ -502,11 +503,12 @@ def process_users(options, account_ids, storage_name, client, host_institute=GEN
     ok_users = []
 
     for vsc_id in sorted(account_ids):
-        user = VscTier2AccountpageUser(vsc_id,
-                                       rest_client=client,
-                                       host_institute=host_institute,
-                                       use_user_cache=use_user_cache)
-        user.dry_run = options.dry_run
+        user = VscTier2AccountpageUser(
+            vsc_id,
+            rest_client=client,
+            host_institute=host_institute,
+            use_user_cache=use_user_cache)
+        user.dry_run = dry_run
 
         try:
             if storage_name == VSC_HOME:

lib/vsc/administration/vo.py

@@ -30,7 +30,7 @@
 
 from vsc.accountpage.wrappers import mkVo, mkVscVoSizeQuota, mkVscAccount, mkVscAutogroup
 from vsc.administration.user import VscTier2AccountpageUser, UserStatusUpdateError
-from vsc.config.base import VSC, VscStorage, VSC_HOME, VSC_DATA, VSC_DATA_SHARED, GENT_PRODUCTION_SCRATCH
+from vsc.config.base import VSC, VscStorage, VSC_HOME, VSC_DATA, VSC_DATA_SHARED, VSC_PRODUCTION_SCRATCH
 from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE, GENT, DATA_KEY, SCRATCH_KEY
 from vsc.filesystem.gpfs import GpfsOperations, GpfsOperationError, PosixOperations
 from vsc.utils.missing import Monoid, MonoidDict
@@ -514,7 +514,7 @@ def update_vo_status(vo, client):
                                         (vo.vo_id, virtual_organisation.status))
 
 
-def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute=None):
+def process_vos(vo_ids, storage_name, client, datestamp, host_institute=None, dry_run=False):
     """Process the virtual organisations.
 
     - make the fileset per VO
@@ -529,7 +529,7 @@ def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute
     for vo_id in sorted(vo_ids):
 
         vo = VscTier2AccountpageVo(vo_id, rest_client=client)
-        vo.dry_run = options.dry_run
+        vo.dry_run = dry_run
 
         try:
             if storage_name in [VSC_HOME]:
@@ -548,7 +548,7 @@ def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute
                 logging.info("Not deploying default VO %s members" % (vo_id,))
                 continue
 
-            if storage_name in GENT_PRODUCTION_SCRATCH:
+            if storage_name in VSC_PRODUCTION_SCRATCH[host_institute]:
                 vo.create_scratch_fileset(storage_name)
                 vo.set_scratch_quota(storage_name)
 
@@ -557,20 +557,21 @@ def process_vos(options, vo_ids, storage_name, client, datestamp, host_institute
                 continue
 
             modified_member_list = client.vo[vo.vo_id].member.modified[datestamp].get()
-            factory = lambda vid: VscTier2AccountpageUser(vid,
-                                                          rest_client=client,
-                                                          host_institute=host_institute,
-                                                          use_user_cache=True)
+            factory = lambda vid: VscTier2AccountpageUser(
+                vid,
+                rest_client=client,
+                host_institute=host_institute,
+                use_user_cache=True)
             modified_members = [factory(a["vsc_id"]) for a in modified_member_list[1]]
 
             for member in modified_members:
                 try:
-                    member.dry_run = options.dry_run
+                    member.dry_run = dry_run
                     if storage_name in [VSC_DATA]:
                         vo.set_member_data_quota(member)  # half of the VO quota
                         vo.create_member_data_dir(member)
 
-                    if storage_name in GENT_PRODUCTION_SCRATCH:
+                    if storage_name in VSC_PRODUCTION_SCRATCH[host_institute]:
                         vo.set_member_scratch_quota(storage_name, member)  # half of the VO quota
                         vo.create_member_scratch_dir(storage_name, member)
 

setup.py

@@ -33,7 +33,7 @@
         'vsc-install >= 0.12.2',
     ],
     'install_requires': [
-        'vsc-accountpage-clients >= 1.2.0',
+        'vsc-accountpage-clients >= 1.3.0',
         'vsc-base >= 2.8.3',
         'vsc-config >= 2.6.0',
         'vsc-filesystems >= 0.40.0',