Skip to content
This repository was archived by the owner on Aug 29, 2018. It is now read-only.

Commit 799b22f

Browse files
author
danehans
committed
Adds Support for Broker API High-Availability
Previously, only a single broker could be deployed. This patch adds support for using HAProxy and Keepalived to load-balance multiple brokers for high-availability and scalability.
1 parent 7b3f4c7 commit 799b22f

8 files changed

+286
-10
lines changed

‎Modulefile

+2
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,6 @@ dependency 'rharrison/lokkit', '>=0.2.0'
1111
dependency 'puppetlabs/ntp', '>=0.1.0'
1212
dependency 'puppetlabs/stdlib', '>=2.6.0'
1313
dependency 'blentz/selinux_types', '>=0.1.0'
14+
dependency 'puppetlabs/haproxy', '>=0.4.1'
15+
dependency 'arioch/keepalived', '>=0.0.10'
1416
dependency 'duritong/sysctl', '>=0.0.1'

‎README.asciidoc

+40-5
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,12 @@ $ d.register_application "appname", "namespace", "node_fqdn"
108108

109109
Choose from the following roles to be configured on this node.
110110

111-
* broker - Installs the broker and console.
112-
* node - Installs the node and cartridges.
113-
* msgserver - Installs ActiveMQ message broker.
114-
* datastore - Installs MongoDB (not sharded/replicated)
115-
* nameserver - Installs a BIND dns server configured with a TSIG key for updates.
111+
* broker - Installs the broker and console.
112+
* node - Installs the node and cartridges.
113+
* msgserver - Installs ActiveMQ message broker.
114+
* datastore - Installs MongoDB (not sharded/replicated)
115+
* nameserver - Installs a BIND dns server configured with a TSIG key for updates.
116+
* load_balancer - Installs HAProxy and Keepalived for Broker API high-availability.
116117

117118
Default: ['broker','node','msgserver','datastore','nameserver']
118119

@@ -233,6 +234,40 @@ for the nameserver IP if none is given.
233234

234235
Default: the current IP (at install)
235236

237+
==== broker_cluster_members
238+
An array of broker hostnames that will be load-balanced for high-availability.
239+
240+
Default: undef
241+
242+
==== broker_cluster_ip_addresses
243+
An array of Broker IP addresses within the load-balanced cluster.
244+
245+
Default: undef
246+
247+
==== broker_virtual_ip_address
248+
The virtual IP address that will front-end the Broker cluster.
249+
250+
Default: undef
251+
252+
==== broker_virtual_hostname
253+
The hostame that represents the Broker API cluster. This name is associated
254+
to broker_virtual_ip_address and added to Named for DNS resolution.
255+
256+
Default: "broker.${domain}"
257+
258+
==== load_balancer_master
259+
Sets the state of the load-balancer. Valid options are true or false.
260+
true sets the load-balancer as the active listener for the Broker cluster
261+
Virtual IP address. Only 1 load_balancer_master is allowed within a Broker cluster.
262+
263+
Default: false
264+
265+
==== load_balancer_auth_password
266+
The password used to secure communication between the load-balancers
267+
within a Broker cluster.
268+
269+
Default: 'changeme'
270+
236271
==== node_ip_addr
237272
This is used for the node to give a public IP, if different from the
238273
one on its NIC.

‎configure_origin.pp.broker_ha_example

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
class { 'openshift_origin' :
2+
# Components to install on this host:
3+
roles => ['broker','nameserver','msgserver','datastore','load_balancer'],
4+
5+
# Broker API Load-Balancer Configuration
6+
broker_cluster_members => [<BROKER1_HOSTNAME>,<BROKER2_HOSTNAME>,<BROKER3_HOSTNAME>],
7+
broker_cluster_ip_addresses => [<BROKER1_IP_ADDR>, <BROKER2_IP_ADDR>, <BROKER3_IP_ADDR>], # Array of Broker Servers
8+
broker_virtual_ip_address => '<BROKER_VIRTUAL_IP>',
9+
load_balancer_master => true, # Only 1 master in a cluster
10+
11+
# BIND / nameserver config
12+
# This is the key for updating the OpenShift BIND server
13+
bind_key => '<DNSSEC_BIND_KEY>',
14+
# The domain under which applications should be created.
15+
domain => 'example.com',
16+
# Apps would be nameserver <app>-<namespace>.example.com
17+
# This also creates hostnames for local components under our domain
18+
register_host_with_nameserver => true,
19+
# Forward requests for other domains (to Google by default)
20+
conf_nameserver_upstream_dns => ['<UPSTREAM_DNS_IP>'],
21+
22+
# NTP Servers for OpenShift hosts to sync time
23+
ntp_servers => ['<NTP_SERVER_FQDN> iburst'],
24+
25+
# The FQDNs of the OpenShift component hosts
26+
broker_hostname => '<BROKER_HOSTNAME>.example.com',
27+
nameserver_hostname => '<BROKER_HOSTNAME>.example.com',
28+
datastore_hostname => '<BROKER_HOSTNAME>.example.com',
29+
msgserver_hostname => '<BROKER_HOSTNAME>.example.com',
30+
31+
# Auth OpenShift users created with htpasswd tool in /etc/openshift/htpasswd
32+
broker_auth_plugin => 'htpasswd',
33+
# Username and password for initial openshift user
34+
openshift_user1 => 'openshift',
35+
openshift_password1 => 'password',
36+
37+
#Enable development mode for more verbose logs
38+
development_mode => true,
39+
}

‎manifests/init.pp

+50-5
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@
2121
# === Parameters
2222
# [*roles*]
2323
# Choose from the following roles to be configured on this node.
24-
# * broker - Installs the broker and console.
25-
# * node - Installs the node and cartridges.
26-
# * msgserver - Installs ActiveMQ message broker.
27-
# * datastore - Installs MongoDB (not sharded/replicated)
28-
# * nameserver - Installs a BIND dns server configured with a TSIG key for updates.
24+
# * broker - Installs the broker and console.
25+
# * node - Installs the node and cartridges.
26+
# * msgserver - Installs ActiveMQ message broker.
27+
# * datastore - Installs MongoDB (not sharded/replicated)
28+
# * nameserver - Installs a BIND dns server configured with a TSIG key for updates.
29+
# * load_balancer - Installs HAProxy and Keepalived for Broker API high-availability.
2930
# Default: ['broker','node','msgserver','datastore','nameserver']
3031
#
32+
# Note: Multiple servers are required when using the load_balancer role.
33+
#
3134
# [*install_method*]
3235
# Choose from the following ways to provide packages:
3336
# none - install sources are already set up when the script executes (default)
@@ -150,6 +153,34 @@
150153
# This is used for the node to record its broker. Also is the default
151154
# for the nameserver IP if none is given.
152155
#
156+
# [*broker_cluster_members*]
157+
# Default: undef
158+
# An array of broker hostnames that will be load-balanced for high-availability.
159+
#
160+
# [*broker_cluster_ip_addresses*]
161+
# Default: undef
162+
# An array of Broker IP addresses within the load-balanced cluster.
163+
#
164+
# [*broker_virtual_ip_address*]
165+
# Default: undef
166+
# The virtual IP address that will front-end the Broker cluster.
167+
#
168+
# [*broker_virtual_hostname*]
169+
# Default: "broker.${domain}"
170+
# The hostame that represents the Broker API cluster. This name is associated
171+
# to broker_virtual_ip_address and added to Named for DNS resolution.
172+
#
173+
# [*load_balancer_master*]
174+
# Default: false
175+
# Sets the state of the load-balancer. Valid options are true or false.
176+
# true sets load_balancer_master as the active listener for the Broker Cluster
177+
# Virtual IP address.
178+
#
179+
# [*load_balancer_auth_password*]
180+
# Default: 'changeme'
181+
# The password used to secure communication between the load-balancers
182+
# within a Broker cluster.
183+
#
153184
# [*node_ip_addr*]
154185
# Default: the current IP (at install)
155186
# This is used for the node to give a public IP, if different from the
@@ -497,6 +528,12 @@
497528
$aws_secret_key = '',
498529
$aws_zone_id = '',
499530
$broker_ip_addr = $ipaddress,
531+
$broker_cluster_members = undef,
532+
$broker_cluster_ip_addresses = undef,
533+
$broker_virtual_ip_address = undef,
534+
$broker_virtual_hostname = "broker.${domain}",
535+
$load_balancer_master = false,
536+
$load_balancer_auth_password = 'changeme',
500537
$node_ip_addr = $ipaddress,
501538
$configure_ntp = true,
502539
$ntp_servers = ['time.apple.com iburst', 'pool.ntp.org iburst', 'clock.redhat.com iburst'],
@@ -563,6 +600,9 @@
563600
if member( $roles, 'datastore' ) {
564601
Class['openshift_origin::role::nameserver'] -> Class['openshift_origin::role::datastore']
565602
}
603+
if member( $roles, 'load_balancer' ) {
604+
Class['openshift_origin::role::nameserver'] -> Class['openshift_origin::role::load_balancer']
605+
}
566606
}
567607

568608
# Anchors for containing the implementation class
@@ -594,6 +634,11 @@
594634
require => Class['openshift_origin::update_conf_files']
595635
}
596636
}
637+
if member( $roles, 'load_balancer' ) {
638+
class{ 'openshift_origin::role::load_balancer':
639+
require => Class['openshift_origin::update_conf_files']
640+
}
641+
}
597642

598643
class{ 'openshift_origin::update_conf_files': }
599644

‎manifests/load_balancer.pp

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# Introduction
2+
# Class used to load-balance brokers in a
3+
# high-availability OpenShift deployment.
4+
#
5+
# Module Dependencies
6+
# duritong/sysctl
7+
# arioch/keepalived
8+
# puppetlabs/haproxy
9+
#
10+
# Example Usage
11+
# class { 'openshift_origin' :
12+
# broker_cluster_members => ['broker01.example.com','broker02.example.com','broker03.example.com'],
13+
# broker_cluster_ip_addresses => ['10.10.10.11','10.10.10.12','10.10.10.13'],
14+
# broker_virtual_ip_address => '10.10.10.10',
15+
# broker_virtual_hostname => 'broker.example.com',
16+
# load_balancer_master => true,
17+
# }
18+
#
19+
class openshift_origin::load_balancer(
20+
$enable = true,
21+
$manage_service = true,
22+
$state_master = $::openshift_origin::load_balancer_master,
23+
$virtual_ipaddress = $::openshift_origin::broker_virtual_ip_address,
24+
$server_names = $::openshift_origin::broker_cluster_members,
25+
$ipaddresses = $::openshift_origin::broker_cluster_ip_addresses,
26+
$interface = $::openshift_origin::conf_node_external_eth_dev,
27+
$http_port = '80',
28+
$ssl_port = '443',
29+
$virtual_router_id = '50',
30+
$auth_pass = $::openshift_origin::load_balancer_auth_password,
31+
32+
) {
33+
34+
include keepalived
35+
36+
if 'broker' and 'load_balancer' in $::openshift_origin::roles {
37+
Class[openshift_origin::plugins::frontend::apache] -> Class['haproxy']
38+
}
39+
40+
if ($state_master == true) {
41+
$priority = '101'
42+
} else {
43+
$priority = '100'
44+
}
45+
46+
# Required by sysctl module
47+
Exec { path => '/usr/bin:/usr/sbin:/bin:/sbin' }
48+
49+
sysctl::value { 'net.ipv4.ip_nonlocal_bind':
50+
value => '1',
51+
}
52+
53+
keepalived::vrrp::instance { $virtual_router_id:
54+
interface => $interface,
55+
priority => $priority,
56+
state => $state_master,
57+
virtual_ipaddress => [$virtual_ipaddress],
58+
virtual_router_id => $virtual_router_id,
59+
auth_type => 'PASS',
60+
auth_pass => $auth_pass,
61+
track_script => ['haproxy'],
62+
}
63+
64+
keepalived::vrrp::script { 'haproxy':
65+
script => '/usr/bin/killall -0 haproxy',
66+
}
67+
68+
class { 'haproxy':
69+
manage_service => $manage_service,
70+
enable => $enable,
71+
defaults_options => {
72+
'log' => 'global',
73+
'option' => 'redispatch',
74+
'retries' => '3',
75+
'timeout' => [
76+
'http-request 10s',
77+
'queue 1m',
78+
'connect 10s',
79+
'client 1m',
80+
'server 1m',
81+
'check 10s',
82+
],
83+
'maxconn' => '8000',
84+
}
85+
}
86+
87+
haproxy::listen { 'broker_http_cluster':
88+
ipaddress => $virtual_ipaddress,
89+
ports => $http_port,
90+
options => {
91+
'option' => ['tcpka', 'tcplog'],
92+
'mode' => 'tcp',
93+
'balance' => 'source',
94+
},
95+
}
96+
97+
haproxy::balancermember { 'http_brokers':
98+
listening_service => 'broker_http_cluster',
99+
server_names => $server_names,
100+
ipaddresses => $ipaddresses,
101+
ports => $http_port,
102+
options => 'check inter 2000 rise 2 fall 5',
103+
}
104+
105+
haproxy::listen { 'broker_ssl_cluster':
106+
ipaddress => $virtual_ipaddress,
107+
ports => $ssl_port,
108+
options => {
109+
'option' => ['tcpka', 'tcplog'],
110+
'mode' => 'tcp',
111+
'balance' => 'source',
112+
},
113+
}
114+
115+
haproxy::balancermember { 'ssl_brokers':
116+
listening_service => 'broker_ssl_cluster',
117+
server_names => $server_names,
118+
ipaddresses => $ipaddresses,
119+
ports => $ssl_port,
120+
options => 'check inter 2000 rise 2 fall 5',
121+
}
122+
}

‎manifests/plugins/frontend/apache.pp

+17
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,23 @@
2727
$servername_conf_template = 'openshift_origin/plugins/frontend/apache/node_servername.conf.erb'
2828
}
2929

30+
if 'broker' and 'load_balancer' in $::openshift_origin::roles {
31+
exec { 'httpd_conf':
32+
path => ['/bin/', '/sbin/', '/usr/bin/', '/usr/sbin/'],
33+
command => "sed -ri \'s/Listen 80/Listen ${openshift_origin::broker_ip_addr}:80/\' /etc/httpd/conf/httpd.conf",
34+
unless => "grep \"Listen ${openshift_origin::broker_ip_addr}:80\" /etc/httpd/conf/httpd.conf",
35+
require => Package['httpd'],
36+
notify => Service['httpd'],
37+
}
38+
exec { 'ssl_conf':
39+
path => ['/bin/', '/sbin/', '/usr/bin/', '/usr/sbin/'],
40+
command => "sed -ri \'s/Listen 443/Listen ${openshift_origin::broker_ip_addr}:443/\' /etc/httpd/conf.d/ssl.conf",
41+
unless => "grep \"Listen ${openshift_origin::broker_ip_addr}:443\" /etc/httpd/conf.d/ssl.conf",
42+
require => Package['httpd'],
43+
notify => Service['httpd'],
44+
}
45+
}
46+
3047
service { 'httpd':
3148
ensure => true,
3249
enable => true,

‎manifests/role/load_balancer.pp

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
class openshift_origin::role::load_balancer inherits openshift_origin::role {
2+
include openshift_origin::load_balancer
3+
4+
openshift_origin::register_dns{ 'register virtual broker dns':
5+
fqdn => $::openshift_origin::broker_virtual_hostname,
6+
require => Class['openshift_origin::load_balancer'],
7+
}
8+
}

‎templates/register_dns.erb

+8
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,11 @@
44
echo update add <%= scope.lookupvar('fqdn') %> 180 A <%= scope.lookupvar('::openshift_origin::node_ip_addr') %>
55
echo send
66
) | nsupdate -y <%= scope.lookupvar('::openshift_origin::domain') %>:<%= scope.lookupvar('::openshift_origin::bind_key') %>
7+
<% if scope.lookupvar('::openshift_origin::load_balancer_master') and scope.lookupvar('::openshift_origin::broker_virtual_ip_address')%>
8+
(
9+
echo server <%= scope.lookupvar('::openshift_origin::nameserver_ip_addr') %>
10+
echo update delete <%= scope.lookupvar('::openshift_origin::broker_virtual_hostname') %> A
11+
echo update add <%= scope.lookupvar('::openshift_origin::broker_virtual_hostname') %> 180 A <%= scope.lookupvar('::openshift_origin::broker_virtual_ip_address') %>
12+
echo send
13+
) | nsupdate -y <%= scope.lookupvar('::openshift_origin::domain') %>:<%= scope.lookupvar('::openshift_origin::bind_key') %>
14+
<% end-%>

0 commit comments

Comments
 (0)