Merge branch 'master' of https://github.com/jupyter-incubator/sparkmagic

remotespark/controllerwidget/addendpointwidget.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2015  [email protected]
 # Distributed under the terms of the Modified BSD License.
 from remotespark.controllerwidget.abstractmenuwidget import AbstractMenuWidget
-from remotespark.utils.utils import get_connection_string, get_connection_string_elements
+from remotespark.livyclientlib.endpoint import Endpoint
 
 
 class AddEndpointWidget(AbstractMenuWidget):
@@ -33,52 +33,21 @@ def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoin
             width=widget_width
         )
 
-        self.conn_string_widget = self.ipywidget_factory.get_text(
-            description='Connection String:',
-            value=get_connection_string(self.address_widget.value, self.user_widget.value, self.password_widget.value),
-            width=widget_width
-        )
-
-        # Sync values
-        def parts_to_conn(name, old_value, new_value):
-            url = self.address_widget.value
-            user = self.user_widget.value
-            password = self.password_widget.value
-            conn_str = get_connection_string(url, user, password)
-            self.conn_string_widget.value = conn_str
-
-        def conn_to_parts(name, old_value, new_value):
-            try:
-                conn_str = self.conn_string_widget.value
-                cso = get_connection_string_elements(conn_str)
-                self.address_widget.value = cso.url
-                self.user_widget.value = cso.username
-                self.password_widget.value = cso.password
-            except ValueError:
-                pass
-
-        self.address_widget.on_trait_change(parts_to_conn, "value")
-        self.user_widget.on_trait_change(parts_to_conn, "value")
-        self.password_widget.on_trait_change(parts_to_conn, "value")
-        self.conn_string_widget.on_trait_change(conn_to_parts, "value")
-
         # Submit widget
         self.submit_widget = self.ipywidget_factory.get_submit_button(
             description='Add endpoint'
         )
 
-        self.children = [self.ipywidget_factory.get_html(value="<br/>", width=widget_width), self.address_widget,
-                         self.user_widget, self.password_widget, self.conn_string_widget,
+        self.children = [self.ipywidget_factory.get_html(value="<br/>", width=widget_width),
+                         self.address_widget, self.user_widget, self.password_widget,
                          self.ipywidget_factory.get_html(value="<br/>", width=widget_width), self.submit_widget]
 
         for child in self.children:
             child.parent_widget = self
 
     def run(self):
-        connection_string = get_connection_string(self.address_widget.value, self.user_widget.value,
-                                                  self.password_widget.value)
-        self.endpoints[self.address_widget.value] = connection_string
-
+        endpoint = Endpoint(self.address_widget.value, self.user_widget.value, self.password_widget.value)
+        self.endpoints[self.address_widget.value] = endpoint
         self.ipython_display.writeln("Added endpoint {}".format(self.address_widget.value))
 
         # We need to call the refresh method because drop down in Tab 2 for endpoints wouldn't refresh with the new

remotespark/controllerwidget/createsessionwidget.py

@@ -48,14 +48,14 @@ def run(self):
             self.ipython_display.send_error("Session properties must be a valid JSON string. Error:\n{}".format(e))
             return
 
-        connection_string = self.endpoints_dropdown_widget.value
+        endpoint = self.endpoints_dropdown_widget.value
         language = self.lang_widget.value
         alias = self.session_widget.value
         skip = False
         properties = conf.get_session_properties(language)
 
         try:
-            self.spark_controller.add_session(alias, connection_string, skip, properties)
+            self.spark_controller.add_session(alias, endpoint, skip, properties)
         except ValueError as e:
             self.ipython_display.send_error("""Could not add session with
 name:

remotespark/controllerwidget/manageendpointwidget.py

@@ -29,16 +29,16 @@ def get_existing_endpoint_widgets(self):
             endpoint_widgets.append(header)
 
             # Endpoints
-            for url, conn_str in self.endpoints.items():
-                endpoint_widgets.append(self.get_endpoint_widget(url, conn_str))
+            for url, endpoint in self.endpoints.items():
+                endpoint_widgets.append(self.get_endpoint_widget(url, endpoint))
 
             endpoint_widgets.append(self.ipywidget_factory.get_html(value="<br/>", width="600px"))
         else:
             endpoint_widgets.append(self.ipywidget_factory.get_html(value="No endpoints yet.", width="600px"))
 
         return endpoint_widgets
 
-    def get_endpoint_widget(self, url, conn_str):
+    def get_endpoint_widget(self, url, endpoint):
         # 600 px
         width = "600px"
         vbox_outter = self.ipywidget_factory.get_vbox()
@@ -47,32 +47,32 @@ def get_endpoint_widget(self, url, conn_str):
         hbox_outter = self.ipywidget_factory.get_hbox()
         hbox_outter_children = []
         try:
-            vbox_left = self.get_endpoint_left(conn_str, url)
-            cleanup_w = self.get_cleanup_button_endpoint(url, conn_str)
+            vbox_left = self.get_endpoint_left(endpoint, url)
+            cleanup_w = self.get_cleanup_button_endpoint(url, endpoint)
 
             hbox_outter_children.append(vbox_left)
             hbox_outter_children.append(cleanup_w)
         except ValueError as e:
             hbox_outter_children.append(self.ipywidget_factory.get_html(value=str(e), width=width))
 
-        hbox_outter_children.append(self.get_delete_button_endpoint(url, conn_str))
+        hbox_outter_children.append(self.get_delete_button_endpoint(url, endpoint))
         hbox_outter.children = hbox_outter_children
 
         vbox_outter.children = [separator, hbox_outter]
 
         return vbox_outter
 
-    def get_endpoint_left(self, conn_str, url):
+    def get_endpoint_left(self, endpoint, url):
         # 400 px
-        info = self.get_info_endpoint_widget(conn_str, url)
-        delete_session_number = self.get_delete_session_endpoint_widget(url, conn_str)
+        info = self.get_info_endpoint_widget(endpoint, url)
+        delete_session_number = self.get_delete_session_endpoint_widget(url, endpoint)
         vbox_left = self.ipywidget_factory.get_vbox(children=[info, delete_session_number], width="400px")
         return vbox_left
 
-    def get_cleanup_button_endpoint(self, url, conn_str):
+    def get_cleanup_button_endpoint(self, url, endpoint):
         def cleanup_on_click(button):
             try:
-                self.spark_controller.cleanup_endpoint(conn_str)
+                self.spark_controller.cleanup_endpoint(endpoint)
             except ValueError as e:
                 self.ipython_display.send_error("Could not clean up endpoint due to error: {}".format(e))
                 return
@@ -83,7 +83,7 @@ def cleanup_on_click(button):
 
         return cleanup_w
 
-    def get_delete_button_endpoint(self, url, conn_str):
+    def get_delete_button_endpoint(self, url, endpoint):
         def delete_on_click(button):
             self.endpoints.pop(url, None)
             self.refresh_method()
@@ -93,13 +93,13 @@ def delete_on_click(button):
 
         return delete_w
 
-    def get_delete_session_endpoint_widget(self, url, conn_str):
+    def get_delete_session_endpoint_widget(self, url, endpoint):
         session_text = self.ipywidget_factory.get_text(description="Session to delete:", value="0", width="50px")
 
         def delete_endpoint(button):
             try:
                 id = session_text.value
-                self.spark_controller.delete_session_by_id(conn_str, id)
+                self.spark_controller.delete_session_by_id(endpoint, id)
                 self.ipython_display.writeln("Deleted session {} at {}".format(id, url))
             except ValueError as e:
                 self.ipython_display.send_error(str(e))
@@ -110,11 +110,11 @@ def delete_endpoint(button):
 
         return self.ipywidget_factory.get_hbox(children=[session_text, button], width="152px")
 
-    def get_info_endpoint_widget(self, conn_str, url):
+    def get_info_endpoint_widget(self, endpoint, url):
         # 400 px
         width = "400px"
 
-        info_sessions = self.spark_controller.get_all_sessions_endpoint_info(conn_str)
+        info_sessions = self.spark_controller.get_all_sessions_endpoint_info(endpoint)
 
         if len(info_sessions) > 0:
             text = "{}:<br/>{}".format(url, "* {}".format("<br/>* ".join(info_sessions)))

remotespark/controllerwidget/managesessionwidget.py

@@ -30,8 +30,8 @@ def get_existing_session_widgets(self):
             session_widgets.append(self.ipywidget_factory.get_html(value="<hr/>", width="600px"))
 
             # Sessions
-            for name, client in client_dict.items():
-                session_widgets.append(self.get_session_widget(name, client.session_id, client.kind, client.status))
+            for name, session in client_dict.items():
+                session_widgets.append(self.get_session_widget(name, session.id, session.kind, session.status))
 
             session_widgets.append(self.ipywidget_factory.get_html(value="<br/>", width="600px"))
         else:
@@ -43,7 +43,7 @@ def get_session_widget(self, name, session_id, kind, state, button=True):
         hbox = self.ipywidget_factory.get_hbox()
 
         name_w = self.ipywidget_factory.get_html(value=name, width="200px", padding="4px")
-        id_w = self.ipywidget_factory.get_html(value=session_id, width="100px", padding="4px")
+        id_w = self.ipywidget_factory.get_html(value=str(session_id), width="100px", padding="4px")
         kind_w = self.ipywidget_factory.get_html(value=kind, width="100px", padding="4px")
         state_w = self.ipywidget_factory.get_html(value=state, width="100px", padding="4px")
 

remotespark/datawidgets/autovizwidget.py

@@ -40,7 +40,7 @@ def __init__(self, df, encoding, renderer=None, ipywidget_factory=None, encoding
             ipython_display = IpythonDisplay()
         self.ipython_display = ipython_display
 
-        self.df = df
+        self.df = self._convert_to_displayable_dataframe(df)
 
         self.encoding = encoding
 
@@ -114,3 +114,13 @@ def on_render(*args):
         button.on_click(on_render)
 
         children.append(button)
+
+    @staticmethod
+    def _convert_to_displayable_dataframe(df):
+        # Don't change the user's dataframe! Make a copy to make these changes.
+        df = df.copy()
+        # Convert all booleans to string because Plotly doesn't know how to plot booleans,
+        # but it does know how to plot strings.
+        bool_columns = list(df.select_dtypes(include=['bool']).columns)
+        df[bool_columns] = df[bool_columns].astype(str)
+        return df

remotespark/datawidgets/plotlygraphs/graphrenderer.py

@@ -13,15 +13,10 @@
 
 class GraphRenderer(object):
 
-    already_initialized = False
-
-    def __init__(self, testing=False):
-        if not testing and not GraphRenderer.already_initialized:
-            init_notebook_mode()
-            GraphRenderer.already_initialized = True
-
     @staticmethod
     def render(df, encoding, output):
+        with output:
+            init_notebook_mode()
         GraphRenderer._get_graph(encoding.chart_type).render(df, encoding, output)
 
     @staticmethod

remotespark/example_config.json

@@ -1,10 +1,4 @@
 {
-  "serialize": false,
-  "serialize_periodically": false,
-  "serialize_period_seconds": 3,
-
-  "default_chart_type": "area",
-
   "kernel_python_credentials" : {
     "username": "",
     "password": "",
@@ -40,10 +34,10 @@
     }
   },
 
-  "execute_timeout_seconds": 3600,
+  "wait_for_idle_timeout_seconds": 15,
   "status_sleep_seconds": 2,
   "statement_sleep_seconds": 2,
-  "create_sql_context_timeout_seconds": 60,
+  "livy_session_startup_timeout_seconds": 60,
 
   "fatal_error_suggestion": "The code failed because of a fatal error:\n\t{}.\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.",
 

remotespark/kernels/kernelmagics.py

@@ -13,10 +13,11 @@
 from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
 
 import remotespark.utils.configuration as conf
+from remotespark.livyclientlib.command import Command
+from remotespark.livyclientlib.endpoint import Endpoint
 from remotespark.livyclientlib.sqlquery import SQLQuery
 from remotespark.magics.sparkmagicsbase import SparkMagicBase
 from remotespark.utils.constants import LANGS_SUPPORTED
-from remotespark.utils.utils import get_connection_string
 
 
 @magics_class
@@ -30,8 +31,7 @@ def __init__(self, shell, data=None):
 
         # In order to set these following 3 properties, call %%_do_not_call_change_language -l language
         self.language = ""
-        self.url = None
-        self.connection_string = None
+        self.endpoint = None
         self.fatal_error = False
         self.fatal_error_message = ""
 
@@ -103,14 +103,14 @@ def local(self, line, cell="", local_ns=None):
 
     @cell_magic
     def info(self, line, cell="", local_ns=None):
-        self.ipython_display.writeln("Endpoint:\n\t{}\n".format(self.url))
+        self.ipython_display.writeln("Endpoint:\n\t{}\n".format(self.endpoint.url))
 
         self.ipython_display.writeln("Current session ID number:\n\t{}\n".format(
                 self.spark_controller.get_session_id_for_client(self.session_name)))
 
         self.ipython_display.writeln("Session configs:\n\t{}\n".format(conf.get_session_properties(self.language)))
 
-        info_sessions = self.spark_controller.get_all_sessions_endpoint_info(self.connection_string)
+        info_sessions = self.spark_controller.get_all_sessions_endpoint_info(self.endpoint)
         self.print_endpoint_info(info_sessions)
 
     @cell_magic
@@ -155,7 +155,7 @@ def configure(self, line, cell="", local_ns=None):
     @cell_magic
     def spark(self, line, cell="", local_ns=None):
         if self._do_not_call_start_session(""):
-            (success, out) = self.spark_controller.run_cell(cell)
+            (success, out) = self.spark_controller.run_command(Command(cell))
             if success:
                 self.ipython_display.write(out)
             else:
@@ -189,7 +189,7 @@ def cleanup(self, line, cell="", local_ns=None):
         if args.force:
             self._do_not_call_delete_session("")
 
-            self.spark_controller.cleanup_endpoint(self.connection_string)
+            self.spark_controller.cleanup_endpoint(self.endpoint)
         else:
             self.ipython_display.send_error("When you clean up the endpoint, all sessions will be lost, including the "
                                             "one used for this notebook. Include the -f parameter if that's your "
@@ -212,7 +212,7 @@ def delete(self, line, cell="", local_ns=None):
                                                 "delete all sessions for this endpoint.".format(id))
                 return None
 
-            self.spark_controller.delete_session_by_id(self.connection_string, session)
+            self.spark_controller.delete_session_by_id(self.endpoint, session)
         else:
             self.ipython_display.send_error("Include the -f parameter if you understand that all statements executed"
                                             "in this session will be lost.")
@@ -233,7 +233,7 @@ def _do_not_call_start_session(self, line, cell="", local_ns=None):
 
             try:
                 self.session_started = True
-                self.spark_controller.add_session(self.session_name, self.connection_string, skip, properties)
+                self.spark_controller.add_session(self.session_name, self.endpoint, skip, properties)
             except Exception as e:
                 self.fatal_error = True
                 self.fatal_error_message = conf.fatal_error_suggestion().format(e)
@@ -269,12 +269,8 @@ def _do_not_call_change_language(self, line, cell="", local_ns=None):
 
     def refresh_configuration(self):
         credentials = getattr(conf, 'kernel_' + self.language + '_credentials')()
-        ret = (credentials['username'], credentials['password'], credentials['url'])
-        assert(ret[2])
-
-        (username, password, url) = ret
-        self.url = url
-        self.connection_string = get_connection_string(url, username, password)
+        (username, password, url) = (credentials['username'], credentials['password'], credentials['url'])
+        self.endpoint = Endpoint(url, username, password)
 
     def get_session_settings(self, line, force):
         line = line.strip()