initial commit

Author: Claudio Fahey

configure-ssh.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# Written by [email protected]
+
+"""%prog [options] host ...
+
+%prog will configure password-less SSH to remote hosts.
+
+Examples:
+  %prog -u root host1
+    This is the simplest usage. It will prompt for the password for root@host1.
+    When it is done, "ssh root@host1" should run without asking for a password.
+
+  %prog -u root host1 host2
+    Same as above but it will repeat for host1 and host2.
+
+  %prog -u root -p mysecretpassword host1 host2
+    To avoid being prompted for a password, specify it with the -p option.
+    You must have sshpass installed for this to work.
+    Understand the risks of using sshpass before using this in a secure setting.
+
+  %prog -u user1 -U centos host1
+    When configuring password-less SSH, use the account centos@host1 to establish the
+    initial SSH connection. This will configure the ~/.ssh directory for user1@host1 to
+    allow password-less SSH to this account.
+
+  %prog -u user1 -U centos -s host1
+    Same as above but centos@host1 will use sudo to access user1@host1's home directory.
+    This is useful for cloud images where password-less SSH already exists for centos@host1 but
+    not other accounts.
+
+  %prog --help
+    View all options."""
+
+import os
+import optparse
+import tempfile
+import shutil
+import os.path
+
+def configure_ssh(host, user, password, configure_as_user, sudo, public_key_path, configure_as_identity, identity):
+    """host can be IP, fqdn, or relative host name"""
+   
+    # Use a dictionary to make it easy to build the various command strings.
+    p = {
+        'password': password, 
+        'user': user, 
+        'host': host, 
+        'configure_as_user': configure_as_user, 
+        'public_key_path': public_key_path, 
+        'configure_as_identity': configure_as_identity,
+        'identity': identity,
+        }
+
+    if p['password']:
+        p['sshpass'] = 'sshpass -p "%(password)s" ' % p
+    else:
+        p['sshpass'] = ''
+
+    if not p['configure_as_user']:
+        p['configure_as_user'] = p['user']
+
+    if sudo:
+        p['sudo'] = 'sudo '
+        p['ssh_opt_tt'] = '-tt '
+    else:
+        p['sudo'] = ''
+        p['ssh_opt_tt'] = ''
+
+    # Note that when ssh -tt is used, read from stdin blocks. Therefore, use head -1 to prevent blocking.
+    p['remote_cmd'] = ((
+        '%(sudo)smkdir -p ~%(user)s/.ssh ' +
+        '; %(sudo)stouch ~%(user)s/.ssh/authorized_keys ' +
+        '; %(sudo)schmod 700 ~%(user)s/.ssh ' +
+        '; %(sudo)schown -R %(user)s ~%(user)s/.ssh ' +
+        '; %(sudo)schown -R :%(user)s ~%(user)s/.ssh ' +
+        '; %(sudo)schmod 600 ~%(user)s/.ssh/authorized_keys ' +
+        '; head -1 - | %(sudo)stee -a ~%(user)s/.ssh/authorized_keys')
+        % p)
+
+    p['ssh_cmd'] = '"%(remote_cmd)s"' % p
+
+    if p['configure_as_identity']:
+        p['ssh_opt_i'] = '-i %(configure_as_identity)s ' % p
+    else:
+        p['ssh_opt_i'] = ''
+
+    if p['identity']:
+        p['ssh_opt_i_test'] = '-i %(identity)s ' % p
+    else:
+        p['ssh_opt_i_test'] = ''
+
+    # Remove host from known_hosts file to avoid problems with IP address reuse
+    cmd = 'ssh-keygen -R %(host)s' % p
+    print('# %s' % cmd)
+    os.system(cmd)
+
+    # Configure password-less SSH
+    cmd = ((
+        'cat %(public_key_path)s ' +
+        '| %(sshpass)sssh -o StrictHostKeyChecking=no %(ssh_opt_tt)s%(ssh_opt_i)s%(configure_as_user)s@%(host)s %(ssh_cmd)s')
+        % p)
+    print('# %s' % cmd)
+    os.system(cmd)
+
+    # Test password-less SSH
+    cmd = 'ssh %(ssh_opt_i_test)s%(user)s@%(host)s "echo -n success: ; hostname"' % p
+    print('# %s' % cmd)
+    os.system(cmd)
+
+def main():
+    parser = optparse.OptionParser(usage=__doc__)
+    parser.add_option('-u', '--user', action='append', dest='users', 
+        help='target user on remote host')
+    parser.add_option('-U', '--configure-as-user', action='store', dest='configure_as_user', 
+        help='configure using this user account on remote host')
+    parser.add_option('-k', '--public-key', action='store', dest='public_key', default='~/.ssh/id_rsa.pub', 
+        help='id_rsa.pub key file to authorize')
+    parser.add_option('-i', '--identity', action='store', dest='identity', default='~/.ssh/id_rsa', 
+        help='id_rsa private key file for local user that should be allowed to SSH into the remote host')
+    parser.add_option('-I', '--configure-as-identity', action='store', dest='configure_as_identity', default='~/.ssh/id_rsa', 
+        help='configure using id_rsa private key file')
+    parser.add_option('-n', '--host', action='append', dest='hosts', 
+        help='name of remote host (-n is optional)')
+    parser.add_option('-p', '--password', action='store', dest='password', 
+        help='password of user on remote host (requires sshpass)')
+    parser.add_option('-s', '--sudo', action='store_true', dest='sudo', 
+        help='use sudo on remote host')
+    options, args = parser.parse_args()
+    hosts = args
+    if options.hosts:
+        hosts += options.hosts
+    if not options.users or not hosts:
+        parser.error('Missing required parameters')
+
+    public_key_path = os.path.expanduser(options.public_key)
+    if not os.path.isfile(public_key_path):
+        parser.error('The public key file %s is missing. Use the command "ssh-keygen -t rsa -b 4096" to create a key pair.' % public_key_path)
+
+    for user in options.users:
+        for host in hosts:
+            configure_ssh(host, user, options.password, options.configure_as_user, options.sudo, public_key_path, options.configure_as_identity, options.identity)
+
+if __name__ == '__main__':
+    main()

mount-nfs.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# Written by [email protected]
+
+"""%prog [options]
+
+%prog will permanently mount an NFS export.
+
+Examples:
+  %prog -m /mnt/home -p nfs-server.example.com:/home
+    This will create the mount point directory, modify /etc/fstab,
+    and mount the NFS export.
+
+  %prog --help
+    View all options."""
+
+import optparse
+import os
+
+def create_mount(mount_point, nfs_path):
+    assert os.system('umount %s ; mkdir -p %s' % (mount_point, mount_point)) == 0
+    assert os.system('grep -v %s /etc/fstab > /tmp/fstab ; cp /tmp/fstab /etc/fstab' % mount_point) == 0
+    assert os.system('echo %s\t%s\tnfs\tnolock,nfsvers=3,tcp,rw,hard,intr,timeo=600,retrans=2,rsize=524288,wsize=524288 >> /etc/fstab' % (nfs_path, mount_point)) == 0
+    assert os.system('mount -a ; ls -lh %s' % mount_point) == 0
+
+def main():
+    parser = optparse.OptionParser(usage=__doc__)
+    parser.add_option('-m', '--mount-point', action='store', dest='mount_point',
+        help='mount point (e.g. /mnt/home)')
+    parser.add_option('-p', '--nfs-path', action='store', dest='nfs_path', 
+        help='NFS path (e.g. nfs-server.example.com:/home)')
+    options, args = parser.parse_args()
+    if not options.mount_point or not options.nfs_path:
+        parser.error('Missing required parameters')
+
+    create_mount(options.mount_point, options.nfs_path)
+
+if __name__ == '__main__':
+    main()

prepare-data-disks.py

@@ -0,0 +1,150 @@
+#!/usr/bin/python -u
+# Written by [email protected]
+
+"""%prog [options] /dev/disk ...
+
+%prog will partition, format, and mount each of the specified disks.
+
+Examples:
+  %prog /dev/sdb
+    This will create a single partition on /dev/sdb, format it using ext4,
+    and then mount it as /grid/0.
+
+  %prog /dev/sdb /dev/sdc
+    Same as above but will repeat with /dev/sdc and mount it is /grid/1.
+
+  %prog -a -m sdb-c
+    Automatically discover the disks that match the pattern /dev/sd[b-z],
+    then partition, format, and mount them.
+
+  %prog -a -m sdb-c -t
+    Automatically discover the disks that match the pattern /dev/sd[b-z].
+    Prints the list of disks but does not take any action to partition,
+    format, or mount the disks. Use the -t option to ensure the list of 
+    disks is correct before preparing them.
+
+  %prog --help
+    View all options.
+
+WARNING! THIS WILL ERASE THE CONTENTS OF THESE DISKS!"""
+
+import os
+import glob
+import platform
+import multiprocessing
+import subprocess
+import optparse
+from time import sleep
+
+def get_disks_to_prepare(discover_method):
+    disks = []
+    if discover_method == 'ecs':
+        disk_list_commands = [
+            'parted -m -s -l | grep /dev/.*:4001GB | cut -d : -f 1 | sort',
+            'parted -m -s -l | grep /dev/.*:6001GB | cut -d : -f 1 | sort',
+            'parted -m -s -l | grep /dev/.*:8002GB | cut -d : -f 1 | sort',
+            ]
+        disks = [subprocess.check_output(cmd, shell=True) for cmd in disk_list_commands]
+        disks = ''.join(disks)
+        disks = disks.rstrip('\n').split('\n')
+        disks = [d for d in disks if d != '']
+    elif discover_method == 'sdb-z':
+        disks = sorted(glob.glob('/dev/sd[b-z]'))
+    elif discover_method == 'vdb-z':
+        disks = sorted(glob.glob('/dev/vd[b-z]'))
+    return disks
+
+def umount_disk(disk_info):
+    disk = disk_info['disk']
+    disk_number = disk_info['disk_number']    
+    print('%s: Umounting disk %d, %s' % (platform.node(), disk_number, disk))
+    os.system('umount %s1' % disk)
+
+def partition_disk(disk_info):
+    disk = disk_info['disk']
+    disk_number = disk_info['disk_number']    
+    print('%s: Partitioning disk %d, %s' % (platform.node(), disk_number, disk))
+    if True:
+        assert os.system('parted -s %s mklabel GPT' % disk) == 0
+        os.system('parted -s %s rm 1' % disk)
+        os.system('parted -s %s rm 2' % disk)
+        # Must sleep to avoid errors updating OS partition table.
+        sleep(0.2)
+        assert os.system('parted -s -a cylinder %s -- mkpart primary ext4 1 -1' % disk) == 0
+
+def format_disk(disk_info):
+    disk = disk_info['disk']
+    disk_number = disk_info['disk_number']    
+    mount = disk_info['mount']    
+    print('%s: Formatting disk %d, %s, %s' % (platform.node(), disk_number, disk, mount))
+    if True:
+        assert os.system('mkfs.ext4 -q -T largefile -m 0 %s1' % disk) == 0
+
+def mount_disk(disk_info):
+    disk = disk_info['disk']
+    disk_number = disk_info['disk_number']    
+    mount = disk_info['mount']    
+    print('%s: Mounting disk %d, %s, %s' % (platform.node(), disk_number, disk, mount))
+    assert os.system('mkdir -p /grid/%d' % disk_number) == 0
+    assert os.system('echo %s1\t%s\text4\tdefaults,noatime\t0\t0 >> /etc/fstab' % (disk, mount)) == 0
+    assert os.system('mount %s' % mount) == 0
+
+def main():
+    parser = optparse.OptionParser(usage=__doc__)
+    parser.add_option('-a', '--auto', action='store_true', dest='auto', 
+        help='automatically discover disks to prepare')
+    parser.add_option('-m', '--discover-method', action='store', dest='discover_method', default='sdb-z',
+        help='method to discover disks to prepare')
+    parser.add_option('-x', '--exclude', action='append', dest='exclude', 
+        help='disk to always exclude (will not be prepared)')
+    parser.add_option('-t', '--test', action='store_true', dest='test', 
+        help='show disks that will be prepared but do not prepare them')
+    parser.add_option('-n', '--disk-count', action='store', dest='disk_count', type='int',
+        help='ensure there are exactly this many disks to prepare')    
+    options, args = parser.parse_args()
+    disks = args
+
+    # Get list of disk partitions to format and mount.
+    if options.auto:
+        disks += get_disks_to_prepare(discover_method=options.discover_method)
+
+    if options.exclude:
+        disks = [d for d in disks if d not in options.exclude]
+
+    disk_info = [{
+        'disk_number': disk_number,     # First disk will be 0
+        'disk': disk,
+        'mount': '/grid/%d' % disk_number
+        } for disk_number, disk in enumerate(disks)]
+
+    print('The following %d disks will be erased:' % len(disks))
+    print('  ' + '\n  '.join(disks))
+
+    if options.disk_count:
+        assert len(disks) == options.disk_count
+
+    if options.test:
+        return 0
+
+    assert os.system('mkdir -p /grid') == 0
+
+    # Remove /grid mounts from fstab.
+    assert os.system('egrep -v "/grid/" /etc/fstab > /tmp/fstab ; cp /tmp/fstab /etc/fstab') == 0
+
+    map(umount_disk, disk_info)
+    map(partition_disk, disk_info)
+    map(umount_disk, disk_info)
+
+    # Format all disks in parallel.
+    pool = multiprocessing.Pool(len(disks))
+    pool.map(format_disk, disk_info)
+    print('Format done.')
+
+    map(mount_disk, disk_info)
+
+    assert os.system('mount | grep /grid/') == 0
+
+    print('%s: prepare_data_disks complete.' % platform.node())
+
+if __name__ == '__main__':
+    main()