Merge remote-tracking branch 'upstream/master'

RogPodge · Feb 4, 2020 · 46276f2 · 46276f2
2 parents 1a2bd91 + f4ab5ef
commit 46276f2
Show file tree

Hide file tree

Showing 117 changed files with 2,946 additions and 821 deletions.
diff --git a/.rat-excludes b/.rat-excludes
@@ -14,7 +14,7 @@ logs/*
 **/dataTables.bootstrap.min.css
 **/dataTables.bootstrap.min.js
 **/jquery.dataTables.min.js
-**/jquery-3.2.1.min.js
+**/jquery-3.4.1.min.js
 **/fonts/**
 **/livy_python_api.egg-info/*
 **/*.txt

diff --git a/.travis.yml b/.travis.yml
@@ -16,7 +16,7 @@
 #
 
 sudo: required
-dist: xenial
+dist: trusty
 language: scala
   - 2.11.12
 

diff --git a/NOTICE b/NOTICE
@@ -1,5 +1,5 @@
 Apache Livy
-Copyright 2018 The Apache Software Foundation
+Copyright 2018 and onwards The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/api/pom.xml b/api/pom.xml
@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
   </parent>
 
   <groupId>org.apache.livy</groupId>
   <artifactId>livy-api</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <properties>

diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>livy-assembly</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <properties>

diff --git a/client-common/pom.xml b/client-common/pom.xml
@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
   </parent>
 
   <groupId>org.apache.livy</groupId>
   <artifactId>livy-client-common</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

diff --git a/client-http/pom.xml b/client-http/pom.xml
@@ -20,12 +20,12 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
   </parent>
 
   <groupId>org.apache.livy</groupId>
   <artifactId>livy-client-http</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

diff --git a/conf/livy.conf.template b/conf/livy.conf.template
@@ -64,7 +64,7 @@
 # livy.server.session.state-retain.sec = 600s
 
 # If livy should impersonate the requesting users when creating a new session.
-# livy.impersonation.enabled = true
+# livy.impersonation.enabled = false
 
 # Logs size livy can cache for each session/batch. 0 means don't cache the logs.
 # livy.cache-log.size = 200
@@ -109,6 +109,8 @@
 # Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
 # configure the state store.
 # livy.server.recovery.mode = off
+# Zookeeper address used for HA and state store. e.g. host1:port1, host2:port2
+# livy.server.zookeeper.url =
 
 # Where Livy should store state to for recovery. Possible values:
 # <empty>: Default. State store disabled.
@@ -119,8 +121,26 @@
 # For filesystem state store, the path of the state store directory. Please don't use a filesystem
 # that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
 # For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
+# If livy.server.recovery.state-store is zookeeper, this config is for back-compatibility,
+# so if both this config and livy.server.zookeeper.url exist,
+# livy uses livy.server.zookeeper.url first.
 # livy.server.recovery.state-store.url =
 
+# The policy of curator connecting to zookeeper.
+# For example, m, n means retry m times and the interval of retry is n milliseconds.
+# Please use the new config: livy.server.zk.retry-policy.
+# Keep this config for back-compatibility.
+# If both this config and livy.server.zk.retry-policy exist,
+# livy uses livy.server.zk.retry-policy first.
+# livy.server.recovery.zk-state-store.retry-policy = 5,100
+
+# The policy of curator connecting to zookeeper.
+# For example, m, n means retry m times and the interval of retry is n milliseconds
+# livy.server.zk.retry-policy =
+
+# The dir in zk to store the data about session.
+# livy.server.recovery.zk-state-store.key-prefix = livy
+
 # If Livy can't find the yarn app within this time, consider it lost.
 # livy.server.yarn.app-lookup-timeout = 120s
 # When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would

diff --git a/core/pom.xml b/core/pom.xml
@@ -22,12 +22,12 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>multi-scala-project-root</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
     <relativePath>../scala/pom.xml</relativePath>
   </parent>
 
   <artifactId>livy-core-parent</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <dependencies>

diff --git a/core/scala-2.11/pom.xml b/core/scala-2.11/pom.xml
@@ -19,13 +19,13 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.livy</groupId>
   <artifactId>livy-core_2.11</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-core-parent</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/coverage/pom.xml b/coverage/pom.xml
@@ -23,11 +23,11 @@
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
   </parent>
 
   <artifactId>livy-coverage-report</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <dependencies>

diff --git a/dev/third-party-missing-license.properties b/dev/third-party-missing-license.properties
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+commons-beanutils--commons-beanutils--1.7.0=Apache License, Version 2.0
+javax.servlet.jsp--jsp-api--2.1=CDDL 1.1
+javax.transaction--jta--1.1=CDDL 1.1
+org.antlr--antlr-runtime--3.4=BSD License
+org.apache.zookeeper--zookeeper--3.4.6=Apache License, Version 2.0
+oro--oro--2.0.8=Apache License, Version 2.0
+asm--asm--3.1=The BSD 3-Clause license
+org.codehaus.jettison--jettison--1.1=Apache License, Version 2.0
+ldapsdk--ldapsdk--4.1=
+javax.servlet--servlet-api--2.5=CDDL 1.1
+javax.transaction--transaction-api--1.1=CDDL 1.0
+javax.servlet--jsp-api--2.0=CDDL 1.1
diff --git a/docs/_data/project.yml b/docs/_data/project.yml
@@ -16,6 +16,6 @@
 # Apache Project configurations
 #
 name: Apache Livy
-version: 0.7.0-incubating-SNAPSHOT
+version: 0.8.0-incubating-SNAPSHOT
 
-podling: true
+podling: true
diff --git a/docs/rest-api.md b/docs/rest-api.md
@@ -305,7 +305,7 @@ Cancel the specified statement in this session.
   <tr><th>Name</th><th>Description</th><th>Type</th></tr>
   <tr>
     <td>msg</td>
-    <td>is always "cancelled"</td>
+    <td>is always "canceled"</td>
     <td>string</td>
   </tr>
 </table>
@@ -590,7 +590,7 @@ A session represents an interactive shell.
   </tr>
   <tr>
     <td>kind</td>
-    <td>Session kind (spark, pyspark, or sparkr)</td>
+    <td>Session kind (spark, pyspark, sparkr, or sql)</td>
     <td><a href="#session-kind">session kind</a></td>
   </tr>
   <tr>
@@ -726,6 +726,21 @@ A statement represents the result of an execution statement.
     <td>The execution output</td>
     <td>statement output</td>
   </tr>
+  <tr>
+     <td>progress</td>
+     <td>The execution progress</td>
+     <td>double</td>
+  </tr>
+  <tr>
+     <td>started</td>
+     <td>The start time of statement code</td>
+     <td>long</td>
+  </tr>
+  <tr>
+     <td>completed</td>
+     <td>The complete time of statement code</td>
+     <td>long</td>
+  </tr> 
 </table>
 
 #### Statement State

diff --git a/examples/pom.xml b/examples/pom.xml
@@ -23,13 +23,13 @@
   <parent>
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <groupId>org.apache.livy</groupId>
   <artifactId>livy-examples</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <properties>

diff --git a/integration-test/pom.xml b/integration-test/pom.xml
@@ -23,11 +23,11 @@
     <groupId>org.apache.livy</groupId>
     <artifactId>livy-main</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>0.7.0-incubating-SNAPSHOT</version>
+    <version>0.8.0-incubating-SNAPSHOT</version>
   </parent>
 
   <artifactId>livy-integration-test</artifactId>
-  <version>0.7.0-incubating-SNAPSHOT</version>
+  <version>0.8.0-incubating-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <properties>
@@ -100,6 +100,16 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <classifier>tests</classifier>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.curator</groupId>
+          <artifactId>curator-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -128,6 +138,13 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-tests</artifactId>
       <classifier>tests</classifier>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
+      </exclusions>
+
     </dependency>
 
     <dependency>
@@ -171,6 +188,26 @@
       <artifactId>scalatra-test_${scala.binary.version}</artifactId>
     </dependency>
 
+    <!-- hadoop-common in test scope needs curator-client, but we want to use the proper version -->
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-client</artifactId>
+      <version>${curator.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <!-- hadoop-common, hadoop-yarn-server-tests and curator needs zookeeper, but we want to use the proper version -->
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>${zookeeper.version}</version>
+    </dependency>
+
   </dependencies>
 
   <build>

diff --git a/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala b/integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala
@@ -114,14 +114,16 @@ class InteractiveIT extends BaseIntegrationTestSuite {
   }
 
   test("application kills session") {
-    withNewSession(Spark) { s =>
+    val noCodeCoverageConf = s"${RSCConf.Entry.TEST_NO_CODE_COVERAGE_ANALYSIS.key()}"
+    withNewSession(Spark, Map(noCodeCoverageConf -> "true")) { s =>
       s.runFatalStatement("System.exit(0)")
     }
   }
 
   test("should kill RSCDriver if it doesn't respond to end session") {
     val testConfName = s"${RSCConf.LIVY_SPARK_PREFIX}${RSCConf.Entry.TEST_STUCK_END_SESSION.key()}"
-    withNewSession(Spark, Map(testConfName -> "true")) { s =>
+    val noCodeCoverageConf = s"${RSCConf.Entry.TEST_NO_CODE_COVERAGE_ANALYSIS.key()}"
+    withNewSession(Spark, Map(testConfName -> "true", noCodeCoverageConf -> "true")) { s =>
       val appId = s.appId()
       s.stop()
       val appReport = cluster.yarnClient.getApplicationReport(appId)