Merge pull request openshift#3969 from jarrpa/glusterfs-registry-too

Merged by openshift-bot

inventory/byo/hosts.byo.native-glusterfs.example

@@ -0,0 +1,51 @@
+# This is an example of a bring your own (byo) host inventory for a cluster
+# with natively hosted, containerized GlusterFS storage.
+#
+# This inventory may be used with the byo/config.yml playbook to deploy a new
+# cluster with GlusterFS storage, which will use that storage to create a
+# volume that will provide backend storage for a hosted Docker registry.
+#
+# This inventory may also be used with byo/openshift-glusterfs/config.yml to
+# deploy GlusterFS storage on an existing cluster. With this playbook, the
+# registry backend volume will be created but the administrator must then
+# either deploy a hosted registry or change an existing hosted registry to use
+# that volume.
+#
+# There are additional configuration parameters that can be specified to
+# control the deployment and state of a GlusterFS cluster. Please see the
+# documentation in playbooks/byo/openshift-glusterfs/README.md and
+# roles/openshift_storage_glusterfs/README.md for additional details.
+
+[OSEv3:children]
+masters
+nodes
+# Specify there will be GlusterFS nodes
+glusterfs
+
+[OSEv3:vars]
+ansible_ssh_user=root
+deployment_type=origin
+# Specify that we want to use GlusterFS storage for a hosted registry
+openshift_hosted_registry_storage_kind=glusterfs
+
+[masters]
+master  node=True storage=True master=True
+
+[nodes]
+master  node=True storage=True master=True openshift_schedulable=False
+# A hosted registry, by default, will only be deployed on nodes labeled
+# "region=infra".
+node0   node=True openshift_node_labels="{'region': 'infra'}" openshift_schedulable=True
+node1   node=True openshift_node_labels="{'region': 'infra'}" openshift_schedulable=True
+node2   node=True openshift_node_labels="{'region': 'infra'}" openshift_schedulable=True
+
+# Specify the glusterfs group, which contains the nodes that will host
+# GlusterFS storage pods. At a minimum, each node must have a
+# "glusterfs_devices" variable defined. This variable is a list of block
+# devices the node will have access to that is intended solely for use as
+# GlusterFS storage. These block devices must be bare (e.g. have no data, not
+# be marked as LVM PVs), and will be formatted.
+[glusterfs]
+node0  glusterfs_devices='[ "/dev/vdb", "/dev/vdc", "/dev/vdd" ]'
+node1  glusterfs_devices='[ "/dev/vdb", "/dev/vdc", "/dev/vdd" ]'
+node2  glusterfs_devices='[ "/dev/vdb", "/dev/vdc", "/dev/vdd" ]'

inventory/byo/hosts.origin.example

@@ -438,9 +438,6 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 #openshift_hosted_registry_storage_openstack_volumeID=3a650b4f-c8c5-4e0a-8ca5-eaee11f16c57
 #openshift_hosted_registry_storage_volume_size=10Gi
 #
-# Native GlusterFS Registry Storage
-#openshift_hosted_registry_storage_kind=glusterfs
-#
 # AWS S3
 # S3 bucket must already exist.
 #openshift_hosted_registry_storage_kind=object

inventory/byo/hosts.ose.example

@@ -438,9 +438,6 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 #openshift_hosted_registry_storage_openstack_volumeID=3a650b4f-c8c5-4e0a-8ca5-eaee11f16c57
 #openshift_hosted_registry_storage_volume_size=10Gi
 #
-# Native GlusterFS Registry Storage
-#openshift_hosted_registry_storage_kind=glusterfs
-#
 # AWS S3
 #
 # S3 bucket must already exist.

playbooks/byo/openshift-cluster/cluster_hosts.yml

@@ -15,6 +15,8 @@ g_nfs_hosts: "{{ groups.nfs | default([]) }}"
 
 g_glusterfs_hosts: "{{ groups.glusterfs | default([]) }}"
 
+g_glusterfs_registry_hosts: "{{ groups.glusterfs_registry | default(g_glusterfs_hosts) }}"
+
 g_all_hosts: "{{ g_master_hosts | union(g_node_hosts) | union(g_etcd_hosts)
                  | union(g_lb_hosts) | union(g_nfs_hosts)
                  | union(g_new_node_hosts)| union(g_new_master_hosts)

playbooks/byo/openshift-glusterfs/README.md

@@ -0,0 +1,98 @@
+# OpenShift GlusterFS Playbooks
+
+These playbooks are intended to enable the use of GlusterFS volumes by pods in
+OpenShift. While they try to provide a sane set of defaults they do cover a
+variety of scenarios and configurations, so read carefully. :)
+
+## Playbook: config.yml
+
+This is the main playbook that integrates GlusterFS into a new or existing
+OpenShift cluster. It will also, if specified, configure a hosted Docker
+registry with GlusterFS backend storage.
+
+This playbook requires the `glusterfs` group to exist in the Ansible inventory
+file. The hosts in this group are the nodes of the GlusterFS cluster.
+
+ * If this is a newly configured cluster each host must have a
+   `glusterfs_devices` variable defined, each of which must be a list of block
+   storage devices intended for use only by the GlusterFS cluster. If this is
+   also an external GlusterFS cluster, you must specify
+   `openshift_storage_glusterfs_is_native=False`. If the cluster is to be
+   managed by an external heketi service you must also specify
+   `openshift_storage_glusterfs_heketi_is_native=False` and
+   `openshift_storage_glusterfs_heketi_url=<URL>` with the URL to the heketi
+   service. All these variables are specified in `[OSEv3:vars]`,
+ * If this is an existing cluster you do not need to specify a list of block
+   devices but you must specify the following variables in `[OSEv3:vars]`:
+   * `openshift_storage_glusterfs_is_missing=False`
+   * `openshift_storage_glusterfs_heketi_is_missing=False`
+
+By default, pods for a native GlusterFS cluster will be created in the
+`default` namespace. To change this, specify
+`openshift_storage_glusterfs_namespace=<other namespace>` in `[OSEv3:vars]`.
+
+To configure the deployment of a Docker registry with GlusterFS backend
+storage, specify `openshift_hosted_registry_storage_kind=glusterfs` in
+`[OSEv3:vars]`. To create a separate GlusterFS cluster for use only by the
+registry, specify a `glusterfs_registry` group that is populated as the
+`glusterfs` is with the nodes for the separate cluster. If no
+`glusterfs_registry` group is specified, the cluster defined by the `glusterfs`
+group will be used.
+
+To swap an existing hosted registry's backend storage for a GlusterFS volume,
+specify `openshift_hosted_registry_storage_glusterfs_swap=True`. To
+additoinally copy any existing contents from an existing hosted registry,
+specify `openshift_hosted_registry_storage_glusterfs_swapcopy=True`.
+
+**NOTE:** For each namespace that is to have access to GlusterFS volumes an
+Enpoints resource pointing to the GlusterFS cluster nodes and a corresponding
+Service resource must be created. If dynamic provisioning using StorageClasses
+is configure, these resources are created automatically in the namespaces that
+require them. This playbook also takes care of creating these resources in the
+namespaces used for deployment.
+
+An example of a minimal inventory file:
+```
+[OSEv3:children]
+masters
+nodes
+glusterfs
+
+[OSEv3:vars]
+ansible_ssh_user=root
+deployment_type=origin
+
+[masters]
+master
+
+[nodes]
+node0
+node1
+node2
+
+[glusterfs]
+node0 glusterfs_devices='[ "/dev/sdb" ]'
+node1 glusterfs_devices='[ "/dev/sdb", "/dev/sdc" ]'
+node2 glusterfs_devices='[ "/dev/sdd" ]'
+```
+
+## Playbook: registry.yml
+
+This playbook is intended for admins who want to deploy a hosted Docker
+registry with GlusterFS backend storage on an existing OpenShift cluster. It
+has all the same requirements and behaviors as `config.yml`.
+
+## Role: openshift_storage_glusterfs
+
+The bulk of the work is done by the `openshift_storage_glusterfs` role. This
+role can handle the deployment of GlusterFS (if it is to be hosted on the
+OpenShift cluster), the registration of GlusterFS nodes (hosted or standalone),
+and (if specified) integration as backend storage for a hosted Docker registry.
+
+See the documentation in the role's directory for further details.
+
+## Role: openshift_hosted
+
+The `openshift_hosted` role recognizes `glusterfs` as a possible storage
+backend for a hosted docker registry. It will also, if configured, handle the
+swap of an existing registry's backend storage to a GlusterFS volume.

playbooks/byo/openshift-glusterfs/config.yml

@@ -0,0 +1,10 @@
+---
+- include: ../openshift-cluster/initialize_groups.yml
+  tags:
+  - always
+
+- include: ../../common/openshift-cluster/std_include.yml
+  tags:
+  - always
+
+- include: ../../common/openshift-glusterfs/config.yml

playbooks/byo/openshift-glusterfs/filter_plugins

@@ -0,0 +1 @@
+../../../filter_plugins

playbooks/byo/openshift-glusterfs/lookup_plugins

@@ -0,0 +1 @@
+../../../lookup_plugins

playbooks/byo/openshift-glusterfs/registry.yml

@@ -0,0 +1,10 @@
+---
+- include: ../openshift-cluster/initialize_groups.yml
+  tags:
+  - always
+
+- include: ../../common/openshift-cluster/std_include.yml
+  tags:
+  - always
+
+- include: ../../common/openshift-glusterfs/registry.yml

playbooks/byo/openshift-glusterfs/roles

@@ -0,0 +1 @@
+../../../roles

playbooks/common/openshift-cluster/evaluate_groups.yml

@@ -155,5 +155,5 @@
       groups: oo_glusterfs_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ g_glusterfs_hosts | default([]) }}"
+    with_items: "{{ g_glusterfs_hosts | union(g_glusterfs_registry_hosts) | default([]) }}"
     changed_when: no

playbooks/common/openshift-glusterfs/config.yml

@@ -12,7 +12,9 @@
     - service: glusterfs_bricks
       port: "49152-49251/tcp"
   roles:
-  - os_firewall
+  - role: os_firewall
+    when:
+    - openshift_storage_glusterfs_is_native | default(True)
 
 - name: Configure GlusterFS
   hosts: oo_first_master

playbooks/common/openshift-glusterfs/registry.yml

@@ -0,0 +1,49 @@
+---
+- include: config.yml
+
+- name: Initialize GlusterFS registry PV and PVC vars
+  hosts: oo_first_master
+  tags: hosted
+  tasks:
+  - set_fact:
+      glusterfs_pv: []
+      glusterfs_pvc: []
+
+  - set_fact:
+      glusterfs_pv:
+      - name: "{{ openshift.hosted.registry.storage.volume.name }}-glusterfs-volume"
+        capacity: "{{ openshift.hosted.registry.storage.volume.size }}"
+        access_modes: "{{ openshift.hosted.registry.storage.access.modes }}"
+        storage:
+          glusterfs:
+            endpoints: "{{ openshift.hosted.registry.storage.glusterfs.endpoints }}"
+            path: "{{ openshift.hosted.registry.storage.glusterfs.path }}"
+            readOnly: "{{ openshift.hosted.registry.storage.glusterfs.readOnly }}"
+      glusterfs_pvc:
+      - name: "{{ openshift.hosted.registry.storage.volume.name }}-glusterfs-claim"
+        capacity: "{{ openshift.hosted.registry.storage.volume.size }}"
+        access_modes: "{{ openshift.hosted.registry.storage.access.modes }}"
+    when: openshift.hosted.registry.storage.glusterfs.swap
+
+- name: Create persistent volumes
+  hosts: oo_first_master
+  tags:
+  - hosted
+  vars:
+    persistent_volumes: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volumes(groups, glusterfs_pv) }}"
+    persistent_volume_claims: "{{ hostvars[groups.oo_first_master.0] | oo_persistent_volume_claims(glusterfs_pvc) }}"
+  roles:
+  - role: openshift_persistent_volumes
+    when: persistent_volumes | union(glusterfs_pv) | length > 0 or persistent_volume_claims | union(glusterfs_pvc) | length > 0
+
+- name: Create Hosted Resources
+  hosts: oo_first_master
+  tags:
+  - hosted
+  pre_tasks:
+  - set_fact:
+      openshift_hosted_router_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}"
+      openshift_hosted_registry_registryurl: "{{ hostvars[groups.oo_first_master.0].openshift.master.registry_url }}"
+    when: "'master' in hostvars[groups.oo_first_master.0].openshift and 'registry_url' in hostvars[groups.oo_first_master.0].openshift.master"
+  roles:
+  - role: openshift_hosted

roles/openshift_facts/library/openshift_facts.py

@@ -2167,7 +2167,9 @@ def get_defaults(self, roles, deployment_type, deployment_subtype):
                         glusterfs=dict(
                             endpoints='glusterfs-registry-endpoints',
                             path='glusterfs-registry-volume',
-                            readOnly=False),
+                            readOnly=False,
+                            swap=False,
+                            swapcopy=True),
                         host=None,
                         access=dict(
                             modes=['ReadWriteMany']

roles/openshift_hosted/README.md

@@ -28,6 +28,14 @@ From this role:
 | openshift_hosted_registry_selector    | region=infra                             | Node selector used when creating registry. The OpenShift registry will only be deployed to nodes matching this selector. |
 | openshift_hosted_registry_cert_expire_days | `730` (2 years)                     | Validity of the certificates in days. Works only with OpenShift version 1.5 (3.5) and later.                             |
 
+If you specify `openshift_hosted_registry_kind=glusterfs`, the following
+variables also control configuration behavior:
+
+| Name                                         | Default value | Description                                                                  |
+|----------------------------------------------|---------------|------------------------------------------------------------------------------|
+| openshift_hosted_registry_glusterfs_swap     | False         | Whether to swap an existing registry's storage volume for a GlusterFS volume |
+| openshift_hosted_registry_glusterfs_swapcopy | True          | If swapping, also copy the current contents of the registry volume           |
+
 Dependencies
 ------------
 

roles/openshift_hosted/tasks/registry/registry.yml

@@ -61,7 +61,7 @@
     name: "{{ openshift_hosted_registry_serviceaccount }}"
     namespace: "{{ openshift_hosted_registry_namespace }}"
 
-- name: Grant the registry serivce account access to the appropriate scc
+- name: Grant the registry service account access to the appropriate scc
   oc_adm_policy_user:
     user: "system:serviceaccount:{{ openshift_hosted_registry_namespace }}:{{ openshift_hosted_registry_serviceaccount }}"
     namespace: "{{ openshift_hosted_registry_namespace }}"
@@ -126,4 +126,4 @@
 
 - include: storage/glusterfs.yml
   when:
-  - openshift.hosted.registry.storage.kind | default(none) == 'glusterfs'
+  - openshift.hosted.registry.storage.kind | default(none) == 'glusterfs' or openshift.hosted.registry.storage.glusterfs.swap

roles/openshift_hosted/tasks/registry/storage/glusterfs.yml

@@ -1,10 +1,18 @@
 ---
+- name: Get registry DeploymentConfig
+  oc_obj:
+    namespace: "{{ openshift_hosted_registry_namespace }}"
+    state: list
+    kind: dc
+    name: "{{ openshift_hosted_registry_name }}"
+  register: registry_dc
+
 - name: Wait for registry pods
   oc_obj:
     namespace: "{{ openshift_hosted_registry_namespace }}"
     state: list
     kind: pod
-    selector: "{{ openshift_hosted_registry_name }}={{ openshift_hosted_registry_namespace }}"
+    selector: "{% for label, value in registry_dc.results.results[0].spec.selector.iteritems() %}{{ label }}={{ value }}{% if not loop.last %},{% endif %}{% endfor %}"
   register: registry_pods
   until:
   - "registry_pods.results.results[0]['items'] | count > 0"
@@ -38,6 +46,39 @@
     mode: "2775"
     recurse: True
 
+- block:
+  - name: Activate registry maintenance mode
+    oc_env:
+      namespace: "{{ openshift_hosted_registry_namespace }}"
+      name: "{{ openshift_hosted_registry_name }}"
+      env_vars:
+      - REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED: 'true'
+
+  - name: Get first registry pod name
+    set_fact:
+      registry_pod_name: "{{ registry_pods.results.results[0]['items'][0].metadata.name }}"
+
+  - name: Copy current registry contents to new GlusterFS volume
+    command: "oc rsync {{ registry_pod_name }}:/registry/ {{ mktemp.stdout }}/"
+    when: openshift.hosted.registry.storage.glusterfs.swapcopy
+
+  - name: Swap new GlusterFS registry volume
+    oc_volume:
+      namespace: "{{ openshift_hosted_registry_namespace }}"
+      name: "{{ openshift_hosted_registry_name }}"
+      vol_name: registry-storage
+      mount_type: pvc
+      claim_name: "{{ openshift.hosted.registry.storage.volume.name }}-glusterfs-claim"
+
+  - name: Deactivate registry maintenance mode
+    oc_env:
+      namespace: "{{ openshift_hosted_registry_namespace }}"
+      name: "{{ openshift_hosted_registry_name }}"
+      state: absent
+      env_vars:
+      - REGISTRY_STORAGE_MAINTENANCE_READONLY_ENABLED: 'true'
+  when: openshift.hosted.registry.storage.glusterfs.swap
+
 - name: Unmount registry volume
   mount:
     state: unmounted