Skip to content

Commit 9d4cf45

Browse files
mneethirajmneethiraj
authored
RANGER-5494: docker setup fix for startup errors in ranger-hive, ranger-tagsync, ranger-solr containers (#855)
1 parent 8e6f395 commit 9d4cf45

File tree

7 files changed

+205
-6
lines changed

7 files changed

+205
-6
lines changed

‎dev-support/ranger-docker/Dockerfile.ranger-tagsync‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --direct
2929
rm -f ${RANGER_HOME}/tagsync/install.properties && \
3030
mkdir -p /opt/ranger/tagsync/data /var/run/ranger /var/log/ranger/tagsync /etc/ranger && \
3131
cp -f ${RANGER_SCRIPTS}/ranger-tagsync-install.properties ${RANGER_HOME}/tagsync/install.properties && \
32-
cp -f ${RANGER_SCRIPTS}/ranger-tagsync-tags.json ${RANGER_HOME}/tagsync/data/tags.json && \
3332
mkdir /etc/init.d || true && \
3433
mkdir /etc/rc2.d || true && \
3534
mkdir /etc/rc3.d || true && \

‎dev-support/ranger-docker/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ Execute following command to build Apache Ranger:
5353
# optional step: a fresh build ensures that the correct jdk version is used
5454
docker compose -f docker-compose.ranger-build.yml build
5555
56-
docker compose -f docker-compose.ranger-build.yml up -d
56+
docker compose -f docker-compose.ranger-build.yml up
5757
~~~
5858
Time taken to complete the build might vary (upto an hour), depending on status of ```${HOME}/.m2``` directory cache.
5959

‎dev-support/ranger-docker/docker-compose.ranger-tagsync.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ services:
1616
- ./scripts/kdc/krb5.conf:/etc/krb5.conf
1717
- ./scripts/hadoop/core-site.xml:/home/ranger/scripts/core-site.xml:ro
1818
- ./dist/version:/home/ranger/dist/version:ro
19-
- ./scripts/tagsync/ranger-tagsync-tags.json:/home/ranger/scripts/ranger-tagsync-tags.json
19+
- ./scripts/tagsync/ranger-tagsync-tags.json:/opt/ranger/tagsync/data/tags.json:ro
2020
- ./scripts/tagsync/ranger-tagsync-install.properties:/opt/ranger/tagsync/install.properties
2121
stdin_open: true
2222
tty: true

‎dev-support/ranger-docker/scripts/hive/hive-log4j2.properties‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
status = INFO
18+
name = HiveLog4j2
19+
packages = org.apache.hadoop.hive.ql.log
20+
21+
# list of properties
22+
# Change to DEBUG for detailed troubleshooting
23+
property.hive.log.level = INFO
24+
property.hive.root.logger = DRFA
25+
property.hive.log.dir = /opt/hive/logs
26+
property.hive.log.file = hiveserver2.log
27+
property.hive.perflogger.log.level = INFO
28+
29+
# list of all appenders
30+
appenders = console, DRFA
31+
32+
# console appender
33+
appender.console.type = Console
34+
appender.console.name = console
35+
appender.console.target = SYSTEM_ERR
36+
appender.console.layout.type = PatternLayout
37+
appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
38+
39+
# daily rolling file appender
40+
appender.DRFA.type = RollingRandomAccessFile
41+
appender.DRFA.name = DRFA
42+
appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file}
43+
# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
44+
appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
45+
appender.DRFA.layout.type = PatternLayout
46+
appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
47+
appender.DRFA.policies.type = Policies
48+
appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
49+
appender.DRFA.policies.time.interval = 1
50+
appender.DRFA.policies.time.modulate = true
51+
appender.DRFA.strategy.type = DefaultRolloverStrategy
52+
appender.DRFA.strategy.max = 30
53+
54+
# list of all loggers
55+
loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth
56+
57+
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
58+
logger.NIOServerCnxn.level = WARN
59+
60+
logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
61+
logger.ClientCnxnSocketNIO.level = WARN
62+
63+
logger.DataNucleus.name = DataNucleus
64+
logger.DataNucleus.level = ERROR
65+
66+
logger.Datastore.name = Datastore
67+
logger.Datastore.level = ERROR
68+
69+
logger.JPOX.name = JPOX
70+
logger.JPOX.level = ERROR
71+
72+
logger.AmazonAws.name=com.amazonaws
73+
logger.AmazonAws.level = INFO
74+
75+
logger.ApacheHttp.name=org.apache.http
76+
logger.ApacheHttp.level = INFO
77+
78+
logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
79+
logger.PerfLogger.level = ${hive.perflogger.log.level}
80+
81+
# Ranger authorization logger
82+
logger.RangerAuth.name = org.apache.ranger
83+
logger.RangerAuth.level = INFO
84+
85+
#
86+
# Hive authentication
87+
# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security
88+
# logger.HiveAuth.level = DEBUG
89+
#
90+
# Security and Kerberos
91+
# logger.SecurityAuth.name = org.apache.hadoop.security
92+
# logger.SecurityAuth.level = DEBUG
93+
94+
# root logger (controls all packages unless overridden above)
95+
rootLogger.level = ${hive.log.level}
96+
rootLogger.appenderRefs = root
97+
rootLogger.appenderRef.root.ref = ${hive.root.logger}

‎dev-support/ranger-docker/scripts/hive/hive-metastore-log4j2.properties‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
status = INFO
18+
name = HiveMetastoreLog4j2
19+
packages = org.apache.hadoop.hive.ql.log
20+
21+
# list of properties
22+
# Change to DEBUG for detailed troubleshooting
23+
property.hive.log.level = INFO
24+
property.hive.root.logger = DRFA
25+
property.hive.log.dir = /opt/hive/logs
26+
property.hive.log.file = metastore.log
27+
property.hive.perflogger.log.level = INFO
28+
29+
# list of all appenders
30+
appenders = console, DRFA
31+
32+
# console appender
33+
appender.console.type = Console
34+
appender.console.name = console
35+
appender.console.target = SYSTEM_ERR
36+
appender.console.layout.type = PatternLayout
37+
appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
38+
39+
# daily rolling file appender
40+
appender.DRFA.type = RollingRandomAccessFile
41+
appender.DRFA.name = DRFA
42+
appender.DRFA.fileName = ${hive.log.dir}/${hive.log.file}
43+
# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
44+
appender.DRFA.filePattern = ${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
45+
appender.DRFA.layout.type = PatternLayout
46+
appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
47+
appender.DRFA.policies.type = Policies
48+
appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
49+
appender.DRFA.policies.time.interval = 1
50+
appender.DRFA.policies.time.modulate = true
51+
appender.DRFA.strategy.type = DefaultRolloverStrategy
52+
appender.DRFA.strategy.max = 30
53+
54+
# list of all loggers
55+
loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp, RangerAuth
56+
57+
logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
58+
logger.NIOServerCnxn.level = WARN
59+
60+
logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
61+
logger.ClientCnxnSocketNIO.level = WARN
62+
63+
logger.DataNucleus.name = DataNucleus
64+
logger.DataNucleus.level = ERROR
65+
66+
logger.Datastore.name = Datastore
67+
logger.Datastore.level = ERROR
68+
69+
logger.JPOX.name = JPOX
70+
logger.JPOX.level = ERROR
71+
72+
logger.AmazonAws.name=com.amazonaws
73+
logger.AmazonAws.level = INFO
74+
75+
logger.ApacheHttp.name=org.apache.http
76+
logger.ApacheHttp.level = INFO
77+
78+
logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
79+
logger.PerfLogger.level = ${hive.perflogger.log.level}
80+
81+
# Ranger authorization logger
82+
logger.RangerAuth.name = org.apache.ranger
83+
logger.RangerAuth.level = INFO
84+
#
85+
# Hive authentication
86+
# logger.HiveAuth.name = org.apache.hadoop.hive.ql.security
87+
# logger.HiveAuth.level = INFO
88+
#
89+
# Security and Kerberos
90+
# logger.SecurityAuth.name = org.apache.hadoop.security
91+
# logger.SecurityAuth.level = INFO
92+
93+
# root logger (controls all packages unless overridden above)
94+
rootLogger.level = ${hive.log.level}
95+
rootLogger.appenderRefs = root
96+
rootLogger.appenderRef.root.ref = ${hive.root.logger}
97+

‎dev-support/ranger-docker/scripts/hive/ranger-hive-setup.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ fi
3535
cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hive-site.xml
3636
cp ${RANGER_SCRIPTS}/hive-site.xml ${HIVE_HOME}/conf/hiveserver2-site.xml
3737

38+
# fix to address error during HiveServer2 startup due to java.lang.NoClassDefFoundError: org/apache/commons/collections/CollectionUtils
39+
cp ${RANGER_HOME}/ranger-hive-plugin/lib/ranger-hive-plugin-impl/commons-collections-3.2.2.jar ${HIVE_HOME}/lib/
40+
3841
mkdir -p ${HADOOP_HOME}/etc/hadoop
3942

4043
cp ${RANGER_SCRIPTS}/core-site.xml ${HADOOP_HOME}/etc/hadoop/core-site.xml

‎dev-support/ranger-docker/scripts/solr/ranger-solr.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818

1919
SOLR_INSTALL_DIR=/opt/solr
2020

21+
export RANGER_SCRIPTS=/home/ranger/scripts
22+
2123
if [ "${KERBEROS_ENABLED}" == "true" ]
2224
then
23-
/home/ranger/scripts/wait_for_keytab.sh HTTP.keytab
24-
/home/ranger/scripts/wait_for_keytab.sh solr.keytab
25-
/home/ranger/scripts/wait_for_testusers_keytab.sh
25+
${RANGER_SCRIPTS}/wait_for_keytab.sh HTTP.keytab
26+
${RANGER_SCRIPTS}/wait_for_keytab.sh solr.keytab
27+
${RANGER_SCRIPTS}/wait_for_testusers_keytab.sh
2628

2729
JAAS_CONFIG="-Djava.security.auth.login.config=/opt/solr/server/etc/jaas.conf"
2830
JAAS_APPNAME="-Dsolr.kerberos.jaas.appname=Client"
@@ -36,6 +38,7 @@ RULE:[2:\$1/\$2@\$0](jhs/.*@EXAMPLE\.COM)s/.*/mapred/\
3638
DEFAULT"
3739

3840
export SOLR_AUTHENTICATION_OPTS="${JAAS_CONFIG} ${JAAS_APPNAME} ${KRB5_CONF} ${KERBEROS_KEYTAB} ${KERBEROS_PRINCIPAL} ${COOKIE_DOMAIN} ${KERBEROS_NAME_RULES}"
41+
export SOLR_AUTH_TYPE=kerberos
3942
fi
4043

4144
if [ ! -e ${SOLR_INSTALL_DIR}/.setupDone ]

0 commit comments

Comments
 (0)