Skip to content

Commit 465d3c0

Browse files
Jenkinsopenstack-gerrit
Jenkins
authored and
openstack-gerrit
committed
Merge "k8s-fedora: Add etcd_volume_size label"
2 parents f055d16 + 2875c97 commit 465d3c0

File tree

13 files changed

+117
-3
lines changed

13 files changed

+117
-3
lines changed

‎doc/source/userguide.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,9 @@ the table are linked to more details elsewhere in the user guide.
337337
+---------------------------------------+--------------------+---------------+
338338
| `docker_volume_type`_ | see below | see below |
339339
+---------------------------------------+--------------------+---------------+
340+
| `etcd_volume_size`_ | etcd storage | 0 |
341+
| | volume size | |
342+
+---------------------------------------+--------------------+---------------+
340343

341344
=======
342345
Cluster
@@ -1060,6 +1063,10 @@ _`admission_control_list`
10601063
The default value corresponds to the one recommended in this doc
10611064
for our current Kubernetes version.
10621065

1066+
_`etcd_volume_size`
1067+
This label sets the size of a volume holding the etcd storage data.
1068+
The default value is 0, meaning the etcd data is not persisted (no volume).
1069+
10631070
External load balancer for services
10641071
-----------------------------------
10651072

‎magnum/drivers/common/templates/environments/no_etcd_volume.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Environment file to not use a cinder volume for etcd storage
2+
resource_registry:
3+
"Magnum::Optional::Etcd::Volume": "OS::Heat::None"
4+
"Magnum::Optional::Etcd::VolumeAttachment": "OS::Heat::None"

‎magnum/drivers/common/templates/environments/with_etcd_volume.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Environment file to use a volume for etcd storage
2+
resource_registry:
3+
"Magnum::Optional::Etcd::Volume": "OS::Cinder::Volume"
4+
"Magnum::Optional::Etcd::VolumeAttachment": "OS::Cinder::VolumeAttachment"

‎magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,38 @@
22

33
. /etc/sysconfig/heat-params
44

5+
if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
6+
7+
attempts=60
8+
while [ ${attempts} -gt 0 ]; do
9+
device_name=$(ls /dev/disk/by-id | grep ${ETCD_VOLUME:0:20}$)
10+
if [ -n "${device_name}" ]; then
11+
break
12+
fi
13+
echo "waiting for disk device"
14+
sleep 0.5
15+
udevadm trigger
16+
let attempts--
17+
done
18+
19+
if [ -z "${device_name}" ]; then
20+
echo "ERROR: disk device does not exist" >&2
21+
exit 1
22+
fi
23+
24+
device_path=/dev/disk/by-id/${device_name}
25+
fstype=$(blkid -s TYPE -o value ${device_path})
26+
if [ "${fstype}" != "xfs" ]; then
27+
mkfs.xfs -f ${device_path}
28+
fi
29+
mkdir -p /var/lib/etcd
30+
echo "${device_path} /var/lib/etcd xfs defaults 0 0" >> /etc/fstab
31+
mount -a
32+
chown -R etcd.etcd /var/lib/etcd
33+
chmod 755 /var/lib/etcd
34+
35+
fi
36+
537
if [ -z "$KUBE_NODE_IP" ]; then
638
# FIXME(yuanying): Set KUBE_NODE_IP correctly
739
KUBE_NODE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)

‎magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ write_files:
1313
KUBE_NODE_IP="$KUBE_NODE_IP"
1414
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
1515
ENABLE_CINDER="$ENABLE_CINDER"
16+
ETCD_VOLUME="$ETCD_VOLUME"
17+
ETCD_VOLUME_SIZE="$ETCD_VOLUME_SIZE"
1618
DOCKER_VOLUME="$DOCKER_VOLUME"
1719
DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
1820
DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"

‎magnum/drivers/heat/k8s_fedora_template_def.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def get_env_files(self, cluster_template, cluster):
8888
env_files = []
8989

9090
template_def.add_priv_net_env_file(env_files, cluster_template)
91+
template_def.add_etcd_volume_env_file(env_files, cluster_template)
9192
template_def.add_volume_env_file(env_files, cluster)
9293
template_def.add_lb_env_file(env_files, cluster_template)
9394
template_def.add_fip_env_file(env_files, cluster_template)

‎magnum/drivers/heat/k8s_template_def.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ def get_params(self, context, cluster_template, cluster, **kwargs):
112112
'admission_control_list',
113113
'prometheus_monitoring',
114114
'grafana_admin_passwd',
115-
'kube_dashboard_enabled']
115+
'kube_dashboard_enabled',
116+
'etcd_volume_size']
116117

117118
for label in label_list:
118119
extra_params[label] = cluster_template.labels.get(label)

‎magnum/drivers/heat/template_def.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,13 @@ def add_volume_env_file(env_files, cluster):
337337
env_files.append(COMMON_ENV_PATH + 'with_volume.yaml')
338338

339339

340+
def add_etcd_volume_env_file(env_files, cluster_template):
341+
if int(cluster_template.labels.get('etcd_volume_size', 0)) < 1:
342+
env_files.append(COMMON_ENV_PATH + 'no_etcd_volume.yaml')
343+
else:
344+
env_files.append(COMMON_ENV_PATH + 'with_etcd_volume.yaml')
345+
346+
340347
def add_fip_env_file(env_files, cluster_template):
341348
if cluster_template.floating_ip_enabled:
342349
env_files.append(COMMON_ENV_PATH + 'enable_floating_ip.yaml')

‎magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,12 @@ parameters:
130130
constraints:
131131
- allowed_values: ["true", "false"]
132132

133+
etcd_volume_size:
134+
type: number
135+
description: >
136+
size of the cinder volume for etcd storage
137+
default: 0
138+
133139
docker_volume_size:
134140
type: number
135141
description: >
@@ -467,6 +473,7 @@ resources:
467473
master_flavor: {get_param: master_flavor}
468474
external_network: {get_param: external_network}
469475
kube_allow_priv: {get_param: kube_allow_priv}
476+
etcd_volume_size: {get_param: etcd_volume_size}
470477
docker_volume_size: {get_param: docker_volume_size}
471478
docker_volume_type: {get_param: docker_volume_type}
472479
docker_storage_driver: {get_param: docker_storage_driver}

‎magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ parameters:
3535
constraints:
3636
- allowed_values: ["true", "false"]
3737

38+
etcd_volume_size:
39+
type: number
40+
description: >
41+
size of a cinder volume to allocate for etcd storage
42+
3843
docker_volume_size:
3944
type: number
4045
description: >
@@ -290,6 +295,8 @@ resources:
290295
"$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
291296
"$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
292297
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
298+
"$ETCD_VOLUME": {get_resource: etcd_volume}
299+
"$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size}
293300
"$DOCKER_VOLUME": {get_resource: docker_volume}
294301
"$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
295302
"$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
@@ -545,6 +552,24 @@ resources:
545552
subnet: { get_param: fixed_subnet }
546553
protocol_port: 2379
547554

555+
######################################################################
556+
#
557+
# etcd storage. This allocates a cinder volume and attaches it
558+
# to the master.
559+
#
560+
561+
etcd_volume:
562+
type: Magnum::Optional::Etcd::Volume
563+
properties:
564+
size: {get_param: etcd_volume_size}
565+
566+
etcd_volume_attach:
567+
type: Magnum::Optional::Etcd::VolumeAttachment
568+
properties:
569+
instance_uuid: {get_resource: kube-master}
570+
volume_id: {get_resource: etcd_volume}
571+
mountpoint: /dev/vdc
572+
548573
######################################################################
549574
#
550575
# docker storage. This allocates a cinder volume and attaches it

‎magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,12 @@ parameters:
128128
constraints:
129129
- allowed_values: ["true", "false"]
130130

131+
etcd_volume_size:
132+
type: number
133+
description: >
134+
size of the cinder volume for etcd storage
135+
default: 0
136+
131137
docker_volume_size:
132138
type: number
133139
description: >

‎magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ def setUp(self):
5555
'prometheus_monitoring': 'False',
5656
'grafana_admin_passwd': 'fake_pwd',
5757
'kube_dashboard_enabled': 'True',
58-
'docker_volume_type': 'lvmdriver-1'},
58+
'docker_volume_type': 'lvmdriver-1',
59+
'etcd_volume_size': '0'},
5960
'tls_disabled': False,
6061
'server_type': 'vm',
6162
'registry_enabled': False,
@@ -158,7 +159,8 @@ def _test_extract_template_definition(
158159
'prometheus_monitoring': 'False',
159160
'grafana_admin_passwd': 'fake_pwd',
160161
'kube_dashboard_enabled': 'True',
161-
'docker_volume_type': 'lvmdriver-1'},
162+
'docker_volume_type': 'lvmdriver-1',
163+
'etcd_volume_size': '0'},
162164
'http_proxy': 'http_proxy',
163165
'https_proxy': 'https_proxy',
164166
'no_proxy': 'no_proxy',
@@ -184,6 +186,7 @@ def _test_extract_template_definition(
184186
'docker_volume_type': 'lvmdriver-1',
185187
'docker_storage_driver': 'devicemapper',
186188
'discovery_url': 'https://discovery.etcd.io/test',
189+
'etcd_volume_size': '0',
187190
'flannel_network_cidr': '10.101.0.0/16',
188191
'flannel_network_subnetlen': '26',
189192
'flannel_backend': 'vxlan',
@@ -218,6 +221,7 @@ def _test_extract_template_definition(
218221
self.assertEqual(expected, definition)
219222
self.assertEqual(
220223
['../../common/templates/environments/no_private_network.yaml',
224+
'../../common/templates/environments/no_etcd_volume.yaml',
221225
'../../common/templates/environments/with_volume.yaml',
222226
'../../common/templates/environments/no_master_lb.yaml',
223227
'../../common/templates/environments/disable_floating_ip.yaml',
@@ -266,6 +270,7 @@ def test_extract_template_definition_with_registry(
266270
'docker_storage_driver': 'devicemapper',
267271
'docker_volume_size': 20,
268272
'docker_volume_type': 'lvmdriver-1',
273+
'etcd_volume_size': '0',
269274
'external_network': 'external_network_id',
270275
'fixed_network': 'fixed_network',
271276
'fixed_subnet': 'fixed_subnet',
@@ -309,6 +314,7 @@ def test_extract_template_definition_with_registry(
309314
self.assertEqual(expected, definition)
310315
self.assertEqual(
311316
['../../common/templates/environments/no_private_network.yaml',
317+
'../../common/templates/environments/no_etcd_volume.yaml',
312318
'../../common/templates/environments/with_volume.yaml',
313319
'../../common/templates/environments/no_master_lb.yaml',
314320
'../../common/templates/environments/disable_floating_ip.yaml',
@@ -366,6 +372,7 @@ def test_extract_template_definition_only_required(
366372
'grafana_admin_passwd': 'fake_pwd',
367373
'kube_dashboard_enabled': 'True',
368374
'docker_volume_type': 'lvmdriver-1',
375+
'etcd_volume_size': '0',
369376
'insecure_registry_url': '10.0.0.1:5000',
370377
'kube_version': 'fake-version',
371378
'magnum_url': 'http://127.0.0.1:9511/v1',
@@ -386,6 +393,7 @@ def test_extract_template_definition_only_required(
386393
self.assertEqual(expected, definition)
387394
self.assertEqual(
388395
['../../common/templates/environments/with_private_network.yaml',
396+
'../../common/templates/environments/no_etcd_volume.yaml',
389397
'../../common/templates/environments/with_volume.yaml',
390398
'../../common/templates/environments/no_master_lb.yaml',
391399
'../../common/templates/environments/disable_floating_ip.yaml',
@@ -432,6 +440,7 @@ def test_extract_template_definition_coreos_with_disovery(
432440
'network_driver': 'network_driver',
433441
'volume_driver': 'volume_driver',
434442
'discovery_url': 'https://discovery.etcd.io/test',
443+
'etcd_volume_size': '0',
435444
'http_proxy': 'http_proxy',
436445
'https_proxy': 'https_proxy',
437446
'no_proxy': 'no_proxy',
@@ -502,6 +511,7 @@ def test_extract_template_definition_coreos_no_discoveryurl(
502511
'network_driver': 'network_driver',
503512
'volume_driver': 'volume_driver',
504513
'discovery_url': 'http://tokentest/h1/h2/h3',
514+
'etcd_volume_size': '0',
505515
'http_proxy': 'http_proxy',
506516
'https_proxy': 'https_proxy',
507517
'no_proxy': 'no_proxy',
@@ -699,6 +709,7 @@ def test_extract_template_definition_without_discovery_url(
699709
'docker_volume_type': 'lvmdriver-1',
700710
'docker_storage_driver': 'devicemapper',
701711
'discovery_url': 'https://address/token',
712+
'etcd_volume_size': '0',
702713
'http_proxy': 'http_proxy',
703714
'https_proxy': 'https_proxy',
704715
'no_proxy': 'no_proxy',
@@ -730,6 +741,7 @@ def test_extract_template_definition_without_discovery_url(
730741
self.assertEqual(expected, definition)
731742
self.assertEqual(
732743
['../../common/templates/environments/no_private_network.yaml',
744+
'../../common/templates/environments/no_etcd_volume.yaml',
733745
'../../common/templates/environments/with_volume.yaml',
734746
'../../common/templates/environments/no_master_lb.yaml',
735747
'../../common/templates/environments/disable_floating_ip.yaml',

‎magnum/tests/unit/drivers/test_template_definition.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,8 @@ def test_k8s_get_params(self, mock_get_output, mock_get_params,
268268
'kube_dashboard_enabled')
269269
docker_volume_type = mock_cluster_template.labels.get(
270270
'docker_volume_type')
271+
etcd_volume_size = mock_cluster_template.labels.get(
272+
'etcd_volume_size')
271273

272274
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
273275

@@ -287,6 +289,7 @@ def test_k8s_get_params(self, mock_get_output, mock_get_params,
287289
'grafana_admin_passwd': grafana_admin_passwd,
288290
'kube_dashboard_enabled': kube_dashboard_enabled,
289291
'docker_volume_type': docker_volume_type,
292+
'etcd_volume_size': etcd_volume_size,
290293
'username': 'fake_user',
291294
'tenant_name': 'fake_tenant',
292295
'magnum_url': mock_osc.magnum_url.return_value,
@@ -345,6 +348,8 @@ def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
345348
'kube_dashboard_enabled')
346349
docker_volume_type = mock_cluster_template.labels.get(
347350
'docker_volume_type')
351+
etcd_volume_size = mock_cluster_template.labels.get(
352+
'etcd_volume_size')
348353

349354
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
350355

@@ -364,6 +369,7 @@ def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
364369
'grafana_admin_passwd': grafana_admin_passwd,
365370
'kube_dashboard_enabled': kube_dashboard_enabled,
366371
'docker_volume_type': docker_volume_type,
372+
'etcd_volume_size': etcd_volume_size,
367373
'username': 'fake_user',
368374
'tenant_name': 'fake_tenant',
369375
'magnum_url': mock_osc.magnum_url.return_value,

0 commit comments

Comments
 (0)