PR for all MapR docker containers

teradatalabs/mapr52-hive/Dockerfile

@@ -0,0 +1,51 @@
+# Copyright 2017 Teradata
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM teradatalabs/mapr52-base
+MAINTAINER Teradata Docker Team <[email protected]>
+
+# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY
+ADD files/*.sh /root/
+ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml
+ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml
+ADD files/supervisord.conf /etc/supervisord.conf
+COPY files/supervisord.d/* /etc/supervisord.d/
+RUN chmod 777 /root/*.sh \
+
+# INSTALL MYSQL SERVER
+  && yum install -y mysql-server mysql-connector-java \
+  && /root/setup.sh \
+
+# INSTALL MAPR HIVE
+  && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \
+  && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
+
+# SETUP FLAT FILE /home/mapr/storagefile
+  && dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10 \
+# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK
+  && echo "/home/mapr/storagefile" > /root/disk.txt
+
+# HDFS PORTS
+EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756
+
+# YARN PORTS
+EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888
+
+# HIVE PORT
+EXPOSE 9083 10000 22
+
+# SOCKS PORT
+EXPOSE 1180
+
+CMD /root/startup.sh

teradatalabs/mapr52-hive/README.md

@@ -0,0 +1,17 @@
+# mapr52-hive 
+
+Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set.
+For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file.
+If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal
+for general correctness testing.
+
+## Run
+
+```
+$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive
+```
+
+## Oracle license
+
+By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
+[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)

teradatalabs/mapr52-hive/capabilities.txt

@@ -0,0 +1,4 @@
+exposes_mapr
+allows_creating_a_table_in_hive
+allows_selecting_from_the_table
+exposes_socks_proxy

teradatalabs/mapr52-hive/files/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# START SSHD AND THE SOCKS PROXY FOR THE HIVE METASTORE
+supervisorctl start sshd
+supervisorctl start socks-proxy
+
+# CONFIGURE MAPR
+/opt/mapr/server/configure.sh  -N mycluster -Z localhost -C localhost -HS localhost -no-autostart
+
+# SETUP DISK FOR MAPR BY RUNNING DISKSETUP
+/opt/mapr/server/disksetup -M -F /root/disk.txt
+
+# CREATE HIVE PROXY USERS
+chmod 755 /opt/mapr/conf/proxy
+
+# START SERVICES
+service mapr-zookeeper start
+service mapr-warden start
+
+# CONFIGURE HIVE
+/opt/mapr/server/configure.sh -R
+
+# WAIT FOR WARDEN TO START ALL THE SERVICES
+sh /root/wardenTracker.sh
+
+# START HTTPFS SERVICES
+maprcli node services -name httpfs -action start -nodes $(hostname)
+
+# RUN HDFS COMMANDS
+hadoop fs -mkdir /user/root /user/hive /user/hdfs /user/hive/warehouse /var /var/mapr /var/mapr/cluster /var/mapr/cluster/yarn /var/mapr/cluster/yarn/rm /var/mapr/cluster/yarn/rm/staging /var/mapr/cluster/yarn/rm/staging/hive
+hadoop fs -chmod 777 /user/hive /user/hdfs /user/hive/warehouse /var/mapr /var/mapr/cluster/yarn/rm/staging/hive

teradatalabs/mapr52-hive/files/conf/core-site.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>
+  		<name>hadoop.proxyuser.mapr.groups</name>
+  		<value>*</value>
+  	<description>Allow the superuser mapr to impersonate any member of any group</description>
+	</property>
+
+	<property>
+  		<name>hadoop.proxyuser.mapr.hosts</name>
+  		<value>*</value>
+  		<description>The superuser can connect from any host to impersonate a user</description>
+	</property>
+
+</configuration>
+

teradatalabs/mapr52-hive/files/conf/hive-site.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+   <property>
+      <name>datanucleus.schema.autoCreateAll</name>
+      <value>true</value>
+      <description>creates necessary schema on a startup if one doesn't exist. set
+         this to false, after creating it once</description>
+   </property>
+
+   <property>
+  	<name>hive.server2.enable.doAs</name>
+  	<value>true</value>
+  	<description>Set this property to enable impersonation in Hive Server 2</description>
+   </property>
+
+   <property>
+  	<name>hive.metastore.execute.setugi</name>
+  	<value>true</value>
+  	<description>Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored.</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionURL</name>
+      <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
+      <description>JDBC connect string for a JDBC metastore</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionDriverName</name>
+      <value>com.mysql.jdbc.Driver</value>
+      <description>Driver class name for a JDBC metastore</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionUserName</name>
+      <value>root</value>
+      <description>username to use against metastore database</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionPassword</name>
+      <value>root</value>
+      <description>password to use against metastore database</description>
+   </property>
+
+   <property>
+      <name>hive.metastore.uris</name>
+      <value>thrift://localhost:9083</value>
+   </property>
+
+</configuration>

teradatalabs/mapr52-hive/files/setup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# SETUP METASTORE
+mysql_install_db
+
+/usr/bin/mysqld_safe &
+sleep 10s
+
+echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
+echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql
+/usr/bin/mysqladmin -u root password 'root'
+
+killall mysqld
+sleep 10s
+mkdir /var/log/mysql/
+chown mysql:mysql /var/log/mysql/

teradatalabs/mapr52-hive/files/socks-proxy.sh

@@ -0,0 +1 @@
+../../../commons/socks-proxy.sh

teradatalabs/mapr52-hive/files/startup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec supervisord -c /etc/supervisord.conf

teradatalabs/mapr52-hive/files/supervisord.conf

@@ -0,0 +1 @@
+../../../commons/supervisord.conf

teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/bootstrap.conf

teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/mysql-metastore.conf

teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/socks-proxy.conf

teradatalabs/mapr52-hive/files/supervisord.d/sshd.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/sshd.conf

teradatalabs/mapr52-hive/files/wardenTracker.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+hname=$(hostname)
+
+Services=0
+
+maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+
+# WAIT FOR CLDB TO START
+while [ $Services -ne 2 ]
+do
+if [ $maprcliReady == 1 ]
+then
+   maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+   Services=0
+else
+   Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+fi
+done
+
+# WAIT FOR NODEMANAGER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR RESOURCEMANAGER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR HIVE METASTORE TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR HIVESERVER2 TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR JOBHISTORYSERVER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done