NDS-998 / 1199 / 1200 / 1201 / 1213 / 1229: Fix Flannel MTU, ingress/…

…smtp configuration, and etcd timeouts (#91) * NDS-1201: Fixed deploy tools ingress configuration * NDS-1200: Fixed deploy tools bug with conditional register * NDS-998: Raised etcd timeouts to hopefully decrease random failures * NDS-1199: Hopefully fixed MTU issues between Flannel and CoreOS * Fixed copypasta typo in task description * Fixed incorrect variable name in roles/ndslabs-api-gui/defaults/main.yml * Fixed syntax error in roles/openstack-system/tasks/main.yml * Changed /media/storage to use ext4 instead of xfs by default * Adjusted playbooks/openstack-delete.yml to clean up known_hosts and boot volumes * Moved 'image' out of openstack group_vars and into site_vars, since this can differ significantly between OpenStack installations/projects * NDS-1213: Removed hardcoded flannel subnet * Removed known_hosts task, since it wasn't working.. removed Docker volume removal, since we no longer have a separate Docker volume * Adjust SDSC site variables (which are for some reason duplicated as group_vars) * NDS-1229: Added nodeSelector to ndslabs-smtp pod
nds-org · Mar 28, 2018 · b85fae3 · b85fae3
1 parent 5505f41
commit b85fae3
Show file tree

Hide file tree

Showing 15 changed files with 89 additions and 31 deletions.
diff --git a/inventory/group_vars/openstack.yml b/inventory/group_vars/openstack.yml
@@ -1,6 +1,5 @@
 ---
 
-image: CoreOSStable1235.6
 sys_vol_size: 40
 key_path: /root/SAVED_AND_SENSITIVE_VOLUME
 security_groups: default

diff --git a/inventory/group_vars/sdsc.yml b/inventory/group_vars/sdsc.yml
@@ -1,12 +1,12 @@
 ---
 
 flavor_small: m1.large              # 20G disk, 8G ram, 2cpu
-flavor_medium: m1.2xlarge           # 20G disk, 16Gram, 8cpu
-flavor_large: m1.4xlarge            # 20G disk, 64Gram, 16cpu
+flavor_medium: m1.large             # 20G disk, 8G ram, 2cpu
+flavor_large: m1.2xlarge            # 20G disk, 32Gram, 8cpu
 
 native_dns: sdsc.edu
 
-ndslabs_domain: "{{ logical_cluster_name }}.sdsc.ndslabs.org" # ???
+ndslabs_domain: "{{ logical_cluster_name }}.ndslabs.org" 
 
-smtp_host: smtp.sdsc.edu
+smtp_host: outbound.ucsd.edu
 smtp_tls: false
diff --git a/inventory/site_vars/ncsa.yml b/inventory/site_vars/ncsa.yml
@@ -18,9 +18,21 @@ flavor_medium: r2.medium        # 40G disk 16Gmem 2cpu
 flavor_large: m4.large     # 80GdiskG disk 64Gmem 24cpu
 
 smtp_host: smtp.ncsa.uiuc.edu
+smtp_port: 25
 smtp_tls: true
 mtu: 1454
 
+flannel_mtu: 1404
+
+# For NDSLabsDev or NDS-hackathon OpenStack project
+image: "CoreOSStable1235.6"
+
+# For NDSLabs OpenStack project
+# image: "CoreOSStable1235"
+
+# For NDS OpenStack Project
+# image: "Container-Linux (1562.1.0)"
+
 network_name: workbench
 
 site_name: ncsa
diff --git a/inventory/site_vars/sdsc.yml b/inventory/site_vars/sdsc.yml
@@ -1,17 +1,21 @@
 ---
 
 flavor_small: m1.large              # 20G disk, 8G ram, 2cpu
-flavor_medium: m1.2xlarge           # 20G disk, 16Gram, 8cpu
-flavor_large: m1.4xlarge            # 20G disk, 64Gram, 16cpu
+flavor_medium: m1.large             # 20G disk, 8G ram, 2cpu
+flavor_large: m1.2xlarge            # 20G disk, 32Gram, 8cpu
 
 native_dns: sdsc.edu
 
-ndslabs_domain: "{{ logical_cluster_name }}.sdsc.ndslabs.org" # ???
+ndslabs_domain: "{{ logical_cluster_name }}.ndslabs.org"
 
-smtp_host: smtp.sdsc.edu
+smtp_host: outbound.ucsd.edu
+smtp_port: 25
 smtp_tls: false
 mtu: 1458
 
-network_name: workbench
+flannel_mtu: 1408
+image: "CoreOSStable1235"
+
+network_name: ndslabs_network
 
 site_name: sdsc
diff --git a/inventory/site_vars/tacc.yml b/inventory/site_vars/tacc.yml
@@ -12,6 +12,9 @@ smtp_host: ''
 smtp_port: ''
 mtu: 8960
 
+flannel_mtu: 8910
+image: "Container-Linux"
+
 network_name: workbench
 
 site_name: tacc
diff --git a/playbooks/k8s-install.yml b/playbooks/k8s-install.yml
@@ -42,8 +42,8 @@
   roles:
     - docker-log-rotation
 
-- name: Fix flannel mtu
-  hosts: coreos
-  roles:
-    - flannel-mtu
+#- name: Fix flannel mtu
+#  hosts: coreos
+#  roles:
+#    - flannel-mtu
 
diff --git a/playbooks/openstack-delete.yml b/playbooks/openstack-delete.yml
@@ -7,6 +7,7 @@
 - name: Remove LoadBalancer public interface
   hosts: publicip
   connection: local
+  ignore_errors: yes
   tasks:
     - os_floating_ip: server={{ logical_cluster_name }}-{{ inventory_hostname }} state=absent reuse=yes wait=yes
 
@@ -26,14 +27,6 @@
         state: absent
         path: "/root/SAVED_AND_SENSITIVE/inventory_cache/{{ inventory_hostname }}"
 
-# delete docker vols
-- name: Remove docker volumes
-  hosts: compute
-  connection: local
-  tasks:
-    - os_volume: state=absent wait=yes display_name="{{ logical_cluster_name }}-{{ inventory_hostname }}-dockervol"
-
-
 # delete storage vols
 - name: Remove storage volumes
   hosts: all
@@ -47,6 +40,14 @@
   connection: local
   tasks:
     - os_volume: state=absent wait=yes display_name="{{ logical_cluster_name }}-{{ inventory_hostname }}-glfs"
+
+# delete boot vols
+- name: Remove boot volumes
+  hosts: all
+  ignore_errors: yes
+  connection: local
+  tasks:
+    - os_volume: state=absent wait=yes display_name="{{ logical_cluster_name }}-{{ inventory_hostname }}-bootvol"
 
 - name: remove ssh host config
   hosts: cluster

diff --git a/playbooks/openstack-provision.yml b/playbooks/openstack-provision.yml
@@ -75,7 +75,7 @@
   hosts: all:&openstack:&coreos
 
   roles: 
-    - { role: openstack-volume-mount, volume_name: 'storagevol', volume_size: "{{ storage_volume_gb }}", mount_point: '/media/storage', filesystem_format: 'xfs' }
+    - { role: openstack-volume-mount, volume_name: 'storagevol', volume_size: "{{ storage_volume_gb }}", mount_point: '/media/storage', filesystem_format: 'ext4' }
 
 # Bind mount /var/lib/docker from storage volume
 - name: Bind mount /var/lib/docker for all nodes

diff --git a/roles/coreos-ndslabs-devsystem/templates/10-docker0.conf.j2 b/roles/coreos-ndslabs-devsystem/templates/10-docker0.conf.j2
@@ -1,3 +1,3 @@
 [Service]
 Environment="DOCKER_OPT_BIP=--bip=10.200.0.1/16"
-Environment="DOCKER_OPT_MTU=--mtu={{ mtu }}"
+Environment="DOCKER_OPT_MTU=--mtu={{ flannel_mtu }}"
diff --git a/roles/flannel-mtu/templates/workbench-flannel.env.j2 b/roles/flannel-mtu/templates/workbench-flannel.env.j2
@@ -1,4 +1,4 @@
 FLANNEL_NETWORK={{ flannel_subnet }}/{{ flannel_prefix }}
 FLANNEL_SUBNET=10.100.25.1/24
-FLANNEL_MTU={{ mtu }}
+FLANNEL_MTU={{ flannel_mtu }}
 FLANNEL_IPMASQ=false
diff --git a/roles/ndslabs-api-gui/defaults/main.yml b/roles/ndslabs-api-gui/defaults/main.yml
@@ -5,8 +5,8 @@ support_email: [email protected]
 webui_git_dropin_repo: ''
 webui_git_dropin_branch: ''
 workbench_name: 'Labs Workbench'
-spec_repo: 'https://github.com/nds-org/ndslabs-specs.git'
-spec_branch: 'master' 
+spec_git_repo: 'https://github.com/nds-org/ndslabs-specs.git'
+spec_git_branch: 'master' 
 smtp_user: ''
 smtp_password: ''
 gmail_user: ''

diff --git a/roles/ndslabs-api-gui/templates/ndslabs-etcd.yaml.j2 b/roles/ndslabs-api-gui/templates/ndslabs-etcd.yaml.j2
@@ -34,6 +34,11 @@ spec:
         - --advertise-client-urls=http://127.0.0.1:4001
         - --data-dir=/var/etcd/data
         name: ndslabs-etcd
+        env:
+        - name: ETCD_HEARTBEAT_INTERVAL
+          value: "1000"
+        - name: ETCD_ELECTION_TIMEOUT
+          value: "10000"
         ports:
         - containerPort: 4001
           name: server

diff --git a/roles/ndslabs-api-gui/templates/ndslabs-ingress.yaml.j2 b/roles/ndslabs-api-gui/templates/ndslabs-ingress.yaml.j2
@@ -1,9 +1,37 @@
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
-  name: ndslabs-ingress
+  name: ndslabs-auth
   annotations:
-    ingress.kubernetes.io/ssl-redirect: "true"
+    "nginx.ingress.kubernetes.io/auth-url": "https://www.{{ ndslabs_domain }}/cauth/auth"
+    "nginx.ingress.kubernetes.io/auth-signin": "https://www.{{ ndslabs_domain }}/login/#/"
+    "nginx.ingress.kubernetes.io/ssl-redirect": "true"
+    "nginx.ingress.kubernetes.io/force-ssl-redirect": "true"
+spec:
+  tls:
+  - hosts:
+    - www.{{ ndslabs_domain }}
+    secretName: ndslabs-tls-secret
+  rules:
+  - host: www.{{ ndslabs_domain }}
+    http:
+      paths:
+      - path: /logs
+        backend:
+          serviceName: ndslabs-webui
+          servicePort: 80
+      - path: /dashboard
+        backend:
+          serviceName: ndslabs-webui
+          servicePort: 80
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: ndslabs-open
+  annotations:
+    "nginx.ingress.kubernetes.io/ssl-redirect": "true"
+    "nginx.ingress.kubernetes.io/force-ssl-redirect": "true"
 spec:
   tls:
   - hosts:

diff --git a/roles/ndslabs-api-gui/templates/ndslabs-smtp.yaml.j2 b/roles/ndslabs-api-gui/templates/ndslabs-smtp.yaml.j2
@@ -45,3 +45,5 @@ spec:
                 name: ndslabs-config
                 key: smtp.gmail_pass
       restartPolicy: Always
+      nodeSelector:
+          ndslabs-role-loadbal: "true"
diff --git a/roles/openstack-system/tasks/main.yml b/roles/openstack-system/tasks/main.yml
@@ -24,7 +24,7 @@
     auto_ip: false
     network: "{{ network_name }}"
     boot_volume: "{{ logical_cluster_name }}-{{ inventory_hostname }}-bootvol"
-  register: os_host
+  register: os_host_nontacc
   when: site_name != "tacc"
 
 - name: OpenStack System (ephemeral instance)
@@ -39,9 +39,13 @@
     key_name: "{{ key_name }}"
     auto_ip: false
     network: "{{ network_name }}"
-  register: os_host
+  register: os_host_tacc
   when: site_name == "tacc"
 
+- name: combine os_host variables
+  set_fact:
+    os_host: "{{ os_host_tacc if site_name == 'tacc' else os_host_nontacc }}"
+
 #- debug: var=os_host
 
 - name: Grab Private IP