Remove spark.stop() and sc.stop() from notebook examples

Author: moomindani

examples/notebooks/delta_dataframe.ipynb

@@ -29,29 +29,11 @@
     "bucket_name = \"<Your S3 bucket name>\"\n",
     "bucket_prefix = \"<Your S3 bucket prefix>\"\n",
     "database_name = \"delta_dataframe\"\n",
+    "database_prefix = f\"{bucket_prefix}/{database_name}\"\n",
+    "database_location = f\"s3://{bucket_name}/{database_prefix}/\"\n",
     "table_name = \"products\"\n",
-    "table_prefix = f\"{bucket_prefix}/{database_name}/{table_name}\"\n",
-    "table_location = f\"s3://{bucket_name}/{table_prefix}\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5db09157",
-   "metadata": {},
-   "source": [
-    "## Initialize SparkSession"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9e319735",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pyspark.sql import SparkSession\n",
-    "spark.stop()\n",
-    "spark = SparkSession.builder.getOrCreate()"
+    "table_prefix = f\"{database_prefix}/{table_name}\"\n",
+    "table_location = f\"s3://{bucket_name}/{table_prefix}/\""
    ]
   },
   {
@@ -77,16 +59,15 @@
     "bucket.objects.filter(Prefix=f\"{table_prefix}/\").delete()\n",
     "\n",
     "## Drop tables in Glue Data Catalog\n",
+    "glue = boto3.client('glue')\n",
     "try:\n",
-    "    glue = boto3.client('glue')\n",
     "    glue.delete_table(DatabaseName=database_name, Name=table_name)\n",
     "except glue.exceptions.EntityNotFoundException:\n",
-    "    print(f\"{database_name}.{table_name} does not exist\")\n",
+    "    print(f\"Table {database_name}.{table_name} does not exist\")\n",
     "try:\n",
-    "    glue = boto3.client('glue')\n",
     "    glue.delete_table(DatabaseName=database_name, Name='testTable')\n",
     "except glue.exceptions.EntityNotFoundException:\n",
-    "    print(f\"{database_name}.testTable does not exist\")\n"
+    "    print(f\"Table {database_name}.testTable does not exist\")\n"
    ]
   },
   {
@@ -97,6 +78,30 @@
     "## Create Delta table with sample data"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2d32110",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    glue = boto3.client('glue')\n",
+    "    res = glue.get_database(Name=database_name)\n",
+    "    print(f\"Database {database_name} exists.\")\n",
+    "    if 'LocationUri' not in res['Database']:\n",
+    "        print(f\"Warning: Database {database_name} does not have Location. You need to configure location in the database.\")\n",
+    "except glue.exceptions.EntityNotFoundException:\n",
+    "    print(f\"Database {database_name} does not exist.\")\n",
+    "    glue = glue.create_database(\n",
+    "        DatabaseInput={\n",
+    "            'Name': database_name,\n",
+    "            'LocationUri': database_location\n",
+    "        }\n",
+    "    )\n",
+    "    print(f\"Created a new database {database_name}.\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -466,19 +471,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Glue Spark",
-   "language": "scala",
-   "name": "glue_scala_kernel"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "python",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "scala",
-   "pygments_lexer": "python3"
+   "display_name": "Glue PySpark",
+   "language": "python",
+   "name": "glue_pyspark"
   },
   "toc-autonumbering": true,
   "toc-showcode": true,

examples/notebooks/delta_sql.ipynb

@@ -29,29 +29,11 @@
     "bucket_name = \"<Your S3 bucket name>\"\n",
     "bucket_prefix = \"<Your S3 bucket prefix>\"\n",
     "database_name = \"delta_sql\"\n",
+    "database_prefix = f\"{bucket_prefix}/{database_name}\"\n",
+    "database_location = f\"s3://{bucket_name}/{database_prefix}/\"\n",
     "table_name = \"products\"\n",
-    "table_prefix = f\"{bucket_prefix}/{database_name}/{table_name}\"\n",
-    "table_location = f\"s3://{bucket_name}/{table_prefix}\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "52f5b844",
-   "metadata": {},
-   "source": [
-    "## Initialize SparkSession"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "42f78b7b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pyspark.sql import SparkSession\n",
-    "spark.stop()\n",
-    "spark = SparkSession.builder.getOrCreate()"
+    "table_prefix = f\"{database_prefix}/{table_name}\"\n",
+    "table_location = f\"s3://{bucket_name}/{table_prefix}/\""
    ]
   },
   {
@@ -444,19 +426,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Glue Spark",
-   "language": "scala",
-   "name": "glue_scala_kernel"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "python",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "scala",
-   "pygments_lexer": "python3"
+   "display_name": "Glue PySpark",
+   "language": "python",
+   "name": "glue_pyspark"
   }
  },
  "nbformat": 4,