Skip to content
This repository was archived by the owner on Aug 29, 2018. It is now read-only.

Commit 3bd11cb

Browse files
author
danehans
committed
Adds Support for Clustering ActiveMQ & Mcollective
Previously, ActiveMQ and the Mcollective were single points of failure. This patch adds support for clustering these services for high-availability and scalability.
1 parent 7b3f4c7 commit 3bd11cb

10 files changed

+270
-4
lines changed

‎README.asciidoc

+25
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,31 @@ Default: ['time.apple.com iburst', 'pool.ntp.org iburst', 'clock.redhat.com ibur
262262
NOTE: Use iburst after every ntp server definition to speed up the
263263
initial synchronization.
264264

265+
==== msgserver_cluster
266+
Default: false
267+
268+
Set to true to cluster ActiveMQ for high-availability and scalability
269+
of OpenShift message queues.
270+
271+
==== msgserver_cluster_members
272+
Default: undef
273+
274+
An array of ActiveMQ server hostnames to be included in the ActiveMQ
275+
cluster. Required when parameter msgserver_cluster is set to true.
276+
277+
==== mcollective_cluster_members
278+
Default: $msgserver_cluster_members
279+
280+
An array of ActiveMQ server hostnames to be included in the ActiveMQ
281+
cluster. Required when parameter msgserver_cluster is set to true.
282+
283+
==== msgserver_password
284+
Default 'changeme'
285+
286+
Password used by ActiveMQ's amquser. The amquser is used to authenticate
287+
ActiveMQ inter-cluster communication. Only used when msgserver_cluster
288+
is true.
289+
265290
==== msgserver_admin_password
266291
This is the admin password for the ActiveMQ admin console, which is
267292
not needed by OpenShift but might be useful in troubleshooting.

‎configure_origin.pp.broker_example

+5
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,9 @@ class { 'openshift_origin' :
3030

3131
#Enable development mode for more verbose logs
3232
development_mode => true,
33+
34+
# Uncomment msgserver_cluster and msgserver_cluster_members for ActiveMQ High-Availability
35+
# Clustering requires a minimum of 3 servers.
36+
#msgserver_cluster => true,
37+
#msgserver_cluster_members => ['<MSGSERVER_HOSTNAME01>', '<MSGSERVER_HOSTNAME02>', '<MSGSERVER_HOSTNAME03>'],
3338
}

‎manifests/firewall/activemq.pp

+6
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,10 @@
22
lokkit::ports { 'ActiveMQ':
33
tcpPorts => [ '61613' ],
44
}
5+
6+
if $::openshift_origin::msgserver_cluster {
7+
lokkit::ports { 'ActiveMQ-Openwire':
8+
tcpPorts => [ '61616' ],
9+
}
10+
}
511
}

‎manifests/init.pp

+30
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,27 @@
173173
# only alphanumeric values in this script as others may cause syntax
174174
# errors depending on context. If non-alphanumeric values are required,
175175
# update them separately after installation.
176+
#
177+
# [*msgserver_cluster*]
178+
# Default: false
179+
# Set to true to cluster ActiveMQ for high-availability and scalability
180+
# of OpenShift message queues.
181+
#
182+
# [*msgserver_cluster_members*]
183+
# Default: undef
184+
# An array of ActiveMQ server hostnames. Required when parameter
185+
# msgserver_cluster is set to true.
186+
#
187+
# [*mcollective_cluster_members*]
188+
# Default: $msgserver_cluster_members
189+
# An array of ActiveMQ server hostnames. Required when parameter
190+
# msgserver_cluster is set to true.
191+
#
192+
# [*msgserver_password*]
193+
# Default 'changeme'
194+
# Password used by ActiveMQ's amquser. The amquser is used to authenticate
195+
# ActiveMQ inter-cluster communication. Only used when msgserver_cluster
196+
# is true.
176197
#
177198
# [*msgserver_admin_password*]
178199
# Default: scrambled
@@ -500,6 +521,10 @@
500521
$node_ip_addr = $ipaddress,
501522
$configure_ntp = true,
502523
$ntp_servers = ['time.apple.com iburst', 'pool.ntp.org iburst', 'clock.redhat.com iburst'],
524+
$msgserver_cluster = false,
525+
$msgserver_cluster_members = undef,
526+
$mcollective_cluster_members = $msgserver_cluster_members,
527+
$msgserver_password = 'changeme',
503528
$msgserver_admin_password = inline_template('<%= require "securerandom"; SecureRandom.base64 %>'),
504529
$mcollective_user = 'mcollective',
505530
$mcollective_password = 'marionette',
@@ -547,6 +572,11 @@
547572
$manage_firewall = true,
548573
){
549574
include openshift_origin::role
575+
576+
if $msgserver_cluster and ! $msgserver_cluster_members and ! $mcollective_cluster_members {
577+
fail('msgserver_cluster_members and mcollective_cluster_members parameters are required when msgserver_cluster is set')
578+
}
579+
550580
if member( $roles, 'nameserver' ) {
551581
class{ 'openshift_origin::role::nameserver':
552582
before => Class['openshift_origin::update_conf_files'],

‎manifests/mcollective_client.pp

+8
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@
2323

2424
# TODO: Replace with MCollective puppet module call
2525

26+
$cluster_members = $::openshift_origin::mcollective_cluster_members
27+
28+
if $cluster_members {
29+
$pool_size = size($cluster_members)
30+
} else {
31+
$pool_size = '1'
32+
}
33+
2634
file { 'mcollective client config':
2735
ensure => present,
2836
path => "${::openshift_origin::params::ruby_scl_path_prefix}/etc/mcollective/client.cfg",

‎manifests/mcollective_server.pp

+8
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@
2121
require => Class['openshift_origin::install_method'],
2222
}
2323

24+
$cluster_members = $::openshift_origin::mcollective_cluster_members
25+
26+
if $cluster_members {
27+
$pool_size = size($cluster_members)
28+
} else {
29+
$pool_size = '1'
30+
}
31+
2432
# Ensure classes are run in order
2533
Class['Openshift_origin::Role'] -> Class['Openshift_origin::Mcollective_server']
2634
Class['Openshift_origin::Update_conf_files'] -> Class['Openshift_origin::Mcollective_server']

‎manifests/msgserver.pp

+10-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
include openshift_origin::firewall::activemq
2222
}
2323

24+
$cluster_members = $::openshift_origin::msgserver_cluster_members
25+
$cluster_remote_members = delete($cluster_members, $::openshift_origin::msgserver_hostname)
26+
2427
package { ['activemq','activemq-client']:
2528
ensure => present,
2629
require => Class['openshift_origin::install_method'],
@@ -50,9 +53,15 @@
5053
require => Package['activemq'],
5154
}
5255

56+
if $::openshift_origin::msgserver_cluster {
57+
$activemq_config_template_real = 'openshift_origin/activemq/activemq-network.xml.erb'
58+
} else {
59+
$activemq_config_template_real = 'openshift_origin/activemq/activemq.xml.erb'
60+
}
61+
5362
file { 'activemq.xml config':
5463
path => '/etc/activemq/activemq.xml',
55-
content => template('openshift_origin/activemq/activemq.xml.erb'),
64+
content => template($activemq_config_template_real),
5665
owner => 'root',
5766
group => 'root',
5867
mode => '0444',

‎templates/activemq/activemq-network.xml.erb

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<beans
18+
xmlns="http://www.springframework.org/schema/beans"
19+
xmlns:amq="http://activemq.apache.org/schema/core"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
22+
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
23+
24+
<!-- Allows us to use system properties as variables in this configuration file -->
25+
<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
26+
<property name="locations">
27+
<value>file:${activemq.conf}/credentials.properties</value>
28+
</property>
29+
</bean>
30+
31+
<!--
32+
The <broker> element is used to configure the ActiveMQ broker.
33+
-->
34+
<broker xmlns="http://activemq.apache.org/schema/core"
35+
brokerName="<%= scope.lookupvar('::openshift_origin::msgserver_hostname') %>"
36+
useJmx="true"
37+
dataDirectory="${activemq.data}"
38+
schedulePeriodForDestinationPurge="60000">
39+
<!--
40+
For better performances use VM cursor and small memory limit.
41+
For more information, see:
42+
http://activemq.apache.org/message-cursors.html
43+
-->
44+
45+
<destinationPolicy>
46+
<policyMap>
47+
<policyEntries>
48+
<policyEntry topic=">" producerFlowControl="false"/>
49+
<policyEntry queue="*.reply.>" gcInactiveDestinations="true"
50+
inactiveTimoutBeforeGC="300000" />
51+
</policyEntries>
52+
</policyMap>
53+
</destinationPolicy>
54+
55+
56+
<!--
57+
The managementContext is used to configure how ActiveMQ is exposed in
58+
JMX. By default, ActiveMQ uses the MBean server that is started by
59+
the JVM. For more information, see:
60+
61+
http://activemq.apache.org/jmx.html
62+
-->
63+
<managementContext>
64+
<managementContext createConnector="false"/>
65+
</managementContext>
66+
67+
<networkConnectors>
68+
<% @cluster_remote_members.each do |cluster_remote_member| -%>
69+
<networkConnector name="<%= scope.lookupvar('::openshift_origin::msgserver_hostname') %>-<%= cluster_remote_member %>-topic" uri="static:(tcp://<%= cluster_remote_member %>:61616)" userName="amquser" password="<%= scope.lookupvar('::openshift_origin::msgserver_password') %>">
70+
<excludedDestinations><queue physicalName=">" /></excludedDestinations>
71+
</networkConnector>
72+
<networkConnector name="<%= scope.lookupvar('::openshift_origin::msgserver_hostname') %>-<%= cluster_remote_member %>-queue" uri="static:(tcp://<%= cluster_remote_member %>:61616)" userName="amquser" password="<%= scope.lookupvar('::openshift_origin::msgserver_password') %>"
73+
conduitSubscriptions="false">
74+
<excludedDestinations><topic physicalName=">" /></excludedDestinations>
75+
</networkConnector>
76+
<% end -%>
77+
</networkConnectors>
78+
79+
<plugins>
80+
<statisticsBrokerPlugin/>
81+
<simpleAuthenticationPlugin>
82+
<users>
83+
<authenticationUser username="<%= scope.lookupvar('::openshift_origin::mcollective_user') %>" password="<%= scope.lookupvar('::openshift_origin::mcollective_password') %>" groups="mcollective,everyone"/>
84+
<authenticationUser username="amquser" password="<%= scope.lookupvar('::openshift_origin::msgserver_password') %>" groups="admins,everyone"/>
85+
<authenticationUser username="admin" password="<%= scope.lookupvar('::openshift_origin::msgserver_admin_password') %>" groups="mcollective,admin,everyone"/>
86+
</users>
87+
</simpleAuthenticationPlugin>
88+
<authorizationPlugin>
89+
<map>
90+
<authorizationMap>
91+
<authorizationEntries>
92+
<authorizationEntry queue=">" write="admins" read="admins" admin="admins" />
93+
<authorizationEntry topic=">" write="admins" read="admins" admin="admins" />
94+
<authorizationEntry topic="mcollective.>" write="mcollective" read="mcollective" admin="mcollective" />
95+
<authorizationEntry queue="mcollective.>" write="mcollective" read="mcollective" admin="mcollective" />
96+
<authorizationEntry topic="ActiveMQ.Advisory.>" read="everyone" write="everyone" admin="everyone"/>
97+
</authorizationEntries>
98+
</authorizationMap>
99+
</map>
100+
</authorizationPlugin>
101+
</plugins>
102+
103+
<!--
104+
The systemUsage controls the maximum amount of space the broker will
105+
use before slowing down producers. For more information, see:
106+
http://activemq.apache.org/producer-flow-control.html
107+
If using ActiveMQ embedded - the following limits could safely be used:
108+
109+
<systemUsage>
110+
<systemUsage>
111+
<memoryUsage>
112+
<memoryUsage limit="20 mb"/>
113+
</memoryUsage>
114+
<storeUsage>
115+
<storeUsage limit="1 gb"/>
116+
</storeUsage>
117+
<tempUsage>
118+
<tempUsage limit="100 mb"/>
119+
</tempUsage>
120+
</systemUsage>
121+
</systemUsage>
122+
-->
123+
<systemUsage>
124+
<systemUsage>
125+
<memoryUsage>
126+
<memoryUsage limit="64 mb"/>
127+
</memoryUsage>
128+
<storeUsage>
129+
<storeUsage limit="100 gb"/>
130+
</storeUsage>
131+
<tempUsage>
132+
<tempUsage limit="50 gb"/>
133+
</tempUsage>
134+
</systemUsage>
135+
</systemUsage>
136+
137+
<!--
138+
The transport connectors expose ActiveMQ over a given protocol to
139+
clients and other brokers. For more information, see:
140+
141+
http://activemq.apache.org/configuring-transports.html
142+
-->
143+
<transportConnectors>
144+
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616"/>
145+
<transportConnector name="stomp" uri="stomp://0.0.0.0:61613"/>
146+
</transportConnectors>
147+
148+
</broker>
149+
150+
<!--
151+
Enable web consoles, REST and Ajax APIs and demos
152+
153+
Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
154+
-->
155+
<import resource="jetty.xml"/>
156+
157+
</beans>
158+
<!-- END SNIPPET: example -->

‎templates/mcollective/mcollective-client.cfg.erb

+10-2
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,17 @@ securityprovider = psk
1111
plugin.psk = unset
1212

1313
connector = activemq
14-
plugin.activemq.pool.size = 1
14+
plugin.activemq.pool.size = <%= @pool_size %>
15+
<% if scope.lookupvar('::openshift_origin::msgserver_cluster') then
16+
@cluster_members.each_with_index do |cluster_member, index| -%>
17+
plugin.activemq.pool.<%= index + 1%>.host = <%= cluster_member %>
18+
plugin.activemq.pool.<%= index + 1%>.port = 61613
19+
plugin.activemq.pool.<%= index + 1%>.user = <%= scope.lookupvar('::openshift_origin::mcollective_user') %>
20+
plugin.activemq.pool.<%= index + 1%>.password = <%= scope.lookupvar('::openshift_origin::mcollective_password') %>
21+
<% end -%>
22+
<% else -%>
1523
plugin.activemq.pool.1.host = <%= scope.lookupvar('::openshift_origin::msgserver_hostname') %>
1624
plugin.activemq.pool.1.port = 61613
1725
plugin.activemq.pool.1.user = <%= scope.lookupvar('::openshift_origin::mcollective_user') %>
1826
plugin.activemq.pool.1.password = <%= scope.lookupvar('::openshift_origin::mcollective_password') %>
19-
27+
<% end -%>

‎templates/mcollective/mcollective-server.cfg.erb

+10-1
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,20 @@ securityprovider = psk
1313
plugin.psk = unset
1414

1515
connector = activemq
16-
plugin.activemq.pool.size = 1
16+
plugin.activemq.pool.size = <%= @pool_size %>
17+
<% if scope.lookupvar('::openshift_origin::msgserver_cluster') then
18+
@cluster_members.each_with_index do |cluster_member, index| -%>
19+
plugin.activemq.pool.<%= index + 1%>.host = <%= cluster_member %>
20+
plugin.activemq.pool.<%= index + 1%>.port = 61613
21+
plugin.activemq.pool.<%= index + 1%>.user = <%= scope.lookupvar('::openshift_origin::mcollective_user') %>
22+
plugin.activemq.pool.<%= index + 1%>.password = <%= scope.lookupvar('::openshift_origin::mcollective_password') %>
23+
<% end -%>
24+
<% else -%>
1725
plugin.activemq.pool.1.host = <%= scope.lookupvar('::openshift_origin::msgserver_hostname') %>
1826
plugin.activemq.pool.1.port = 61613
1927
plugin.activemq.pool.1.user = <%= scope.lookupvar('::openshift_origin::mcollective_user') %>
2028
plugin.activemq.pool.1.password = <%= scope.lookupvar('::openshift_origin::mcollective_password') %>
29+
<% end -%>
2130

2231
# Facts
2332
factsource = yaml

0 commit comments

Comments
 (0)