Simple HBase log appender implementation with checkstyle corrections

doc/Programming-guide/Key-platform-features/Data-collection/HBase-log-appender/index.md

@@ -0,0 +1,119 @@
+* TOC
+{:toc}
+
+The HBase log appender is used to transfer logs from the [Operations service]({{root_url}}Glossary/#operations-service) to the [HBase](https://hbase.apache.org/) database. This log appender was developed under HBase version 0.98.6.
+
+## Create HBase Log Appender
+
+To create an HBase log appender for your application using the [Administration UI]({{root_url}}Glossary/#administration-ui):
+
+1. Log in to the **Administration UI** page as a [tenant developer]({{root_url}}Glossary/#tenant-developer).
+
+2. Click **Applications** and open the **Log appenders** page of your application.
+Click **Add log appender**.
+
+3. On the **Log appender details** page, enter the necessary information and set the **Type** field to **HBase**.
+
+	![Create hbase log appender](attach/create-hbase-log-appender-admin-ui.png)
+
+>**NOTE:** The field "Log metadata" is not currently working.
+{:.note}
+
+4. Fill in the **Configuration** section for your log appender and click **Add**.
+See [Configure log appender](#configure-log-appender).
+
+	![HBase log appender configuration](attach/hbase-log-appender-config1.png)
+	![](attach/hbase-log-appender-config-zookeeper.png)
+	![](attach/hbase-log-appender-config2.png)
+
+## Configure log appender
+
+The HBase log appender configuration must match [this Avro schema](/server/appenders/hbase-appender/src/main/avro/HBaseAppenderConfiguration.avsc).
+
+You can configure the following log appender settings:
+
+* **Zookeeper Quorum** -- list of Zookeeper hosts responsible for HBase cluster connection. For more than one host, use commas to separate the hosts. 
+* **Keyspace name** -- HBase namespace used to prefix the data table. The namespace can be previously set in HBase. If it is not, the log appender will create it.
+* **Table name** -- HBase table name.
+* **Column families** -- specify and configure column families for the HBase table.
+* **Column mapping** -- mapping of specific log data to appropriate columns. Use the checkboxes to make any field part of the row key.
+
+
+## Playing with HBase log appender
+The example below uses the data collection demo from [My First Kaa Application](/doc/Programming-guide/Your-first-Kaa-application/index.md). However, it extends the number of fields in the log schema. The log appender will send data to Kaa and then persist it to HBase. Some selection queries will be demonstrated using the persisted data.
+
+Below is the log schema for the application.
+
+```json
+{
+    "type":"record",
+    "name":"Data",
+    "namespace":"org.kaaproject.kaa.scheme.sample",
+    "fields":[
+        {
+            "name":"id",
+            "type":"String"
+        },    	
+        {
+            "name":"temperature",
+            "type":"int"
+        },
+        {
+            "name":"timestamp",
+            "type":"long"
+        },
+        {
+            "name":"location",
+            "type":"String"
+        }
+    ],
+    "displayName":"Logging scheme"
+}
+```
+To play around with the HBase log appender:
+
+1. Follow the application instructions from [My First Kaa Application](/doc/Programming-guide/Your-first-Kaa-application/index.md) and use the log schema described above.
+
+2. Log in to the **Administration UI** as a tenant developer, open the **Log appenders** page of **My First Kaa Application** and click **Add log appender**.
+
+3. Follow the HBase log appender configuration settings presented on the figures from [Create HBase Log Appender](#create-hbase-log-appender).
+
+4. Run the application as described in [My First Kaa Application](/doc/Programming-guide/Your-first-Kaa-application/index.md).
+
+5. The console will display the following messages.
+
+		My First Kaa Application started
+		Default sample period: 1
+		Sampled temperature: 28
+		Sampled temperature: 26 
+		Sampled temperature: 27 
+		Sampled temperature: 26 
+		Sampled temperature: 28 
+		...
+
+6. To verify that your logs have been persisted to HBase, open the HBase shell from the terminal using the command below.
+
+    ```bash
+    hbase shell
+    ```
+
+    Then run
+
+    ```bash
+    scan 'kaa:myfirstapplication'
+    ```
+
+7. The following output will be displayed.
+
+    ```bash
+     ROW                        COLUMN+CELL
+     sensorID+1523616858341	column=info:loc,	timestamp=1523616858753,	value=sensor_location
+     sensorID+1523616858341	column=info:temp,	timestamp=1523616858753,	value=28
+     sensorID+1523616858341	column=meta:id,		timestamp=1523616858753,	value=sensorID
+     sensorID+1523616858341	column=meta:ts,		timestamp=1523616858753,	value=1523616858341
+     sensorID+1523616859327	column=info:loc,	timestamp=1523616859578,	value=sensor_location
+     sensorID+1523616859327	column=info:temp,	timestamp=1523616859578,	value=26
+     sensorID+1523616859327	column=meta:id,		timestamp=1523616859578,	value=sensorID
+     sensorID+1523616859327	column=meta:ts,		timestamp=1523616859578,	value=1523616859327
+     ...
+    ```

server/appenders/hbase-appender/.checkstyle

@@ -1,7 +1,18 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<fileset-config file-format-version="1.2.0" simple-config="true" sync-formatter="false">
-  <fileset name="all" enabled="true" check-config-name="Google Checks" local="false">
-    <file-match-pattern match-pattern="." include-pattern="true"/>
+<fileset-config file-format-version="1.2.0" simple-config="false" sync-formatter="false">
+  <local-check-config name="maven-checkstyle-plugin checkstyle" location="jar:file:/home/joao/.m2/repository/com/puppycrawl/tools/checkstyle/7.0/checkstyle-7.0.jar!/google_checks.xml" type="remote" description="maven-checkstyle-plugin configuration checkstyle">
+    <property name="checkstyle.cache.file" value="${project_loc}/target/checkstyle-cachefile"/>
+    <property name="checkstyle.header.file" value="/home/joao/workspaceLogOnly/.metadata/.plugins/org.eclipse.core.resources/.projects/hbase-appender/com.basistech.m2e.code.quality.checkstyleConfigurator/checkstyle-header-checkstyle.txt"/>
+  </local-check-config>
+  <fileset name="java-sources-checkstyle" enabled="true" check-config-name="maven-checkstyle-plugin checkstyle" local="true">
+    <file-match-pattern match-pattern="^src/main/java/.*\.java" include-pattern="true"/>
+    <file-match-pattern match-pattern="^src/main/java.*.*gen/.*" include-pattern="false"/>
+    <file-match-pattern match-pattern="^src/main/java.*^target/generated-sources" include-pattern="false"/>
+    <file-match-pattern match-pattern="^src/main/java.*.*org/kaaproject/kaa/schema/.*" include-pattern="false"/>
+    <file-match-pattern match-pattern="^src/main/java.*.*org/kaaproject/kaa/common/dto/ctl/.*" include-pattern="false"/>
+    <file-match-pattern match-pattern="^src/main/java.*.*org/kaaproject/kaa/server/common/Version\.java" include-pattern="false"/>
+    <file-match-pattern match-pattern="^src/main/resources.*\.properties" include-pattern="true"/>
+    <file-match-pattern match-pattern="^src/test/resources.*\.properties" include-pattern="true"/>
   </fileset>
 </fileset-config>

server/appenders/hbase-appender/pom.xml

@@ -25,8 +25,12 @@
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<main.dir>${basedir}/../../..</main.dir>
+		<hbase.version>0.98.6-hadoop2</hbase.version>
+		<protobuf.version>2.5.0</protobuf.version>
+		<commons-configuration.version>1.6</commons-configuration.version>
 	</properties>
 
+
 	<dependencyManagement>
 		<dependencies>
 			<dependency>
@@ -68,39 +72,38 @@
 		<dependency>
 			<groupId>org.apache.hbase</groupId>
 			<artifactId>hbase-client</artifactId>
-			<version>0.98.6-hadoop2</version>
+			<version>${hbase.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.apache.hbase</groupId>
 			<artifactId>hbase-common</artifactId>
-			<version>0.98.6-hadoop2</version>
+			<version>${hbase.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.hbase</groupId>
 			<artifactId>hbase-protocol</artifactId>
-			<version>0.98.6-hadoop2</version>
+			<version>${hbase.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-common</artifactId>
-			<version>2.4.1</version>
 		</dependency>
 		<dependency>
 			<groupId>commons-configuration</groupId>
 			<artifactId>commons-configuration</artifactId>
-			<version>1.6</version>
+			<version>${commons-configuration.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-auth</artifactId>
-			<version>2.4.1</version>
+			<version>${hadoop.version}</version>
 		</dependency>
 
 		<dependency>
 			<groupId>com.google.protobuf</groupId>
 			<artifactId>protobuf-java</artifactId>
-			<version>2.5.0</version>
+			<version>${protobuf.version}</version>
 		</dependency>
 
 		<dependency>
@@ -118,7 +121,7 @@
 			<plugin>
 				<groupId>org.apache.avro</groupId>
 				<artifactId>avro-maven-plugin</artifactId>
-				<version>1.7.7</version>
+				<version>${avro.version}</version>
 				<executions>
 					<execution>
 						<phase>generate-sources</phase>
@@ -136,8 +139,7 @@
 						</configuration>
 					</execution>
 				</executions>
-			</plugin>
-            <!--
+			</plugin>       
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-jar-plugin</artifactId>
@@ -146,7 +148,7 @@
 						<exclude>**/logback.xml</exclude>
 					</excludes>
 				</configuration>
-			</plugin>  -->
+			</plugin> 
             <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-checkstyle-plugin</artifactId>

server/appenders/hbase-appender/src/main/avro/HBaseAppenderConfiguration.avsc

@@ -118,10 +118,10 @@
                                 "type":"enum",
                                 "symbols":[
                                     "NONE",
-				    				"DIFF",
+				    "DIFF",
                                     "FAST_DIFF",
                                     "PREFIX",
-				    				"PREFIX_TREE"
+				    "PREFIX_TREE"
                                 ]
                             }
                         },
@@ -137,30 +137,30 @@
                                     "NONE",
                                     "GZ",
                                     "LZ4",
-				    				"LZO",
-				    				"SNAPPY"
+				    "LZO",
+				    "SNAPPY"
                                 ]
                             }
                         },
 			{
                             "name":"inMemory",
                             "displayName":"In Memory",
                             "weight":0.2,
-			    			"by_default":false,
+			    "by_default":false,
                             "type":"boolean"
                         },
 			{
                             "name":"keepDeletedCells",
                             "displayName":"Keep Deleted Cells",
                             "weight":0.2,
-			    			"by_default":false,
+			    "by_default":false,
                             "type":"boolean"
                         },	
 			{
                             "name":"blockCache",
                             "displayName":"Block Cache",
                             "weight":0.2,
-			    			"by_default":true,
+			    "by_default":true,
                             "type":"boolean"
                         }   
                     ]
@@ -201,7 +201,7 @@
                                     "FLOAT",
                                     "INT",
                                     "TEXT",
-				    				"ARRAY"
+				    "ARRAY"
                                 ]
                             }
                         },

server/appenders/hbase-appender/src/main/java/org/kaaproject/kaa/server/appenders/hbase/appender/HBaseLogEventDao.java

@@ -83,8 +83,8 @@ public HBaseLogEventDao(HBaseAppenderConfiguration config)  {
 
 
       LOG.info("HBase Admin Started");
-    } catch (Exception e) {
-      LOG.error("Ops!", e);
+    } catch (Exception ex) {
+      LOG.error("ERROR - Unable to establish connection with HBase", ex);
     }
   }
 
@@ -95,7 +95,7 @@ public String createHbTable() {
     //Get table and keyspace names
     String table  = configuration.getTableName().toString().toLowerCase().trim();
     String keyspace = configuration.getKeyspace().toString().toLowerCase().trim();
-    LOG.info("Starting creation of table:  {}",keyspace + ":" + table);
+    LOG.info("Starting creation of table:  {}", keyspace + ":" + table);
 
     // Creates the namespace and/or table in case of not existing.
 
@@ -131,8 +131,8 @@ public String createHbTable() {
       //Format of hbase tables -  <namespace>:<table_name>
       namespacedTable = keyspace  + ":" + table;
 
-    } catch (Exception e) {
-      LOG.error("Ops",e);
+    } catch (Exception ex) {
+      LOG.error("ERROR creating table",ex);
     }
     return namespacedTable;
   }

server/appenders/hbase-appender/src/main/java/org/kaaproject/kaa/server/appenders/hbase/config/gen/Algorithm.java

@@ -3,18 +3,11 @@
  * 
  * DO NOT EDIT DIRECTLY
  */
-
 package org.kaaproject.kaa.server.appenders.hbase.config.gen;  
-
 @SuppressWarnings("all")
 @org.apache.avro.specific.AvroGenerated
 public enum Algorithm { 
-  NONE, GZ, LZ4, LZO, SNAPPY;
-  public static final org.apache.avro.Schema SCHEMA$ = 
-      new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":"
-          + "\"Algorithm\",\"namespace\":\"org.kaaproject.kaa.server.appenders.hbase"
-          + ".config.gen\",\"symbols\":[\"NONE\",\"GZ\",\"LZ4\",\"LZO\",\"SNAPPY\"]}");
-  public static org.apache.avro.Schema getClassSchema() {
-    return SCHEMA$;
-  }
+  NONE, GZ, LZ4, LZO, SNAPPY  ;
+  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Algorithm\",\"namespace\":\"org.kaaproject.kaa.server.appenders.hbase.config.gen\",\"symbols\":[\"NONE\",\"GZ\",\"LZ4\",\"LZO\",\"SNAPPY\"]}");
+  public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
 }

server/appenders/hbase-appender/src/main/java/org/kaaproject/kaa/server/appenders/hbase/config/gen/BloomFilterType.java

@@ -3,19 +3,11 @@
  * 
  * DO NOT EDIT DIRECTLY
  */
-
 package org.kaaproject.kaa.server.appenders.hbase.config.gen;  
-
 @SuppressWarnings("all")
 @org.apache.avro.specific.AvroGenerated
 public enum BloomFilterType { 
-  NONE, ROW, ROWCOL;
-  public static final org.apache.avro.Schema SCHEMA$ = 
-      new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":"
-          + "\"BloomFilterType\",\"namespace\":\"org.kaaproject.kaa.server.appenders."
-          + "hbase.config.gen\",\"symbols\":[\"NONE\",\"ROW\",\"ROWCOL\"]}");
-
-  public static org.apache.avro.Schema getClassSchema() {
-    return SCHEMA$;
-  }
+  NONE, ROW, ROWCOL  ;
+  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"BloomFilterType\",\"namespace\":\"org.kaaproject.kaa.server.appenders.hbase.config.gen\",\"symbols\":[\"NONE\",\"ROW\",\"ROWCOL\"]}");
+  public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
 }