Install_Apache_sedona.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instalación Apache Sedona\n",
    "1. Iniciar el cluster\n",
    "2. ejecutar la celda inferior ( lo cual va a descargar archivos necesarios para la instalación"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "7c2dd829-1d5b-41a0-acc0-3b8dc3fd34c9",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
       "\n",
       "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
       " 45 28.3M   45 12.9M    0     0  46.2M      0 --:--:-- --:--:-- --:--:-- 46.1M\n",
       "100 28.3M  100 28.3M    0     0  53.5M      0 --:--:-- --:--:-- --:--:-- 53.4M\n",
       "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
       "\n",
       "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
       "100 2576k  100 2576k    0     0  25.9M      0 --:--:-- --:--:-- --:--:-- 25.9M\n",
       "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
       "                                 Dload  Upload   Total   Spent    Left  Speed\n",
       "\n",
       "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
       " 26  419k   26  111k    0     0  1137k      0 --:--:-- --:--:-- --:--:-- 1126k\n",
       "100  419k  100  419k    0     0  3996k      0 --:--:-- --:--:-- --:--:-- 3959k\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n 45 28.3M   45 12.9M    0     0  46.2M      0 --:--:-- --:--:-- --:--:-- 46.1M\n100 28.3M  100 28.3M    0     0  53.5M      0 --:--:-- --:--:-- --:--:-- 53.4M\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n100 2576k  100 2576k    0     0  25.9M      0 --:--:-- --:--:-- --:--:-- 25.9M\n  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n\n  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n 26  419k   26  111k    0     0  1137k      0 --:--:-- --:--:-- --:--:-- 1126k\n100  419k  100  419k    0     0  3996k      0 --:--:-- --:--:-- --:--:-- 3959k\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%sh \n",
    "# Create JAR directory for Sedona\n",
    "mkdir -p /dbfs/FileStore/jars/sedona/1.2.1-incubating\n",
    "\n",
    "# Download the dependencies from Maven into DBFS\n",
    "curl -o /dbfs/FileStore/jars/sedona/1.2.1-incubating/geotools-wrapper-1.1.0-25.2.jar \"https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/1.1.0-25.2/geotools-wrapper-1.1.0-25.2.jar\"\n",
    "\n",
    "curl -o /dbfs/FileStore/jars/sedona/1.2.1-incubating/sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar \"https://repo1.maven.org/maven2/org/apache/sedona/sedona-python-adapter-3.0_2.12/1.2.1-incubating/sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar\"\n",
    "\n",
    "curl -o /dbfs/FileStore/jars/sedona/1.2.1-incubating/sedona-viz-3.0_2.12-1.2.1-incubating.jar \"https://repo1.maven.org/maven2/org/apache/sedona/sedona-viz-3.0_2.12/1.2.1-incubating/sedona-viz-3.0_2.12-1.2.1-incubating.jar\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Verificación\n",
    "3. verificar que si se hayan descargado los archivos "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "f6121935-d20b-4b04-872e-2402c12a619a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>path</th><th>name</th><th>size</th></tr></thead><tbody><tr><td>dbfs:/FileStore/jars/sedona/1.2.1-incubating/geotools-wrapper-1.1.0-25.2.jar</td><td>geotools-wrapper-1.1.0-25.2.jar</td><td>29767549</td></tr><tr><td>dbfs:/FileStore/jars/sedona/1.2.1-incubating/sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar</td><td>sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar</td><td>2638338</td></tr><tr><td>dbfs:/FileStore/jars/sedona/1.2.1-incubating/sedona-viz-3.0_2.12-1.2.1-incubating.jar</td><td>sedona-viz-3.0_2.12-1.2.1-incubating.jar</td><td>429742</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         "dbfs:/FileStore/jars/sedona/1.2.1-incubating/geotools-wrapper-1.1.0-25.2.jar",
         "geotools-wrapper-1.1.0-25.2.jar",
         29767549
        ],
        [
         "dbfs:/FileStore/jars/sedona/1.2.1-incubating/sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar",
         "sedona-python-adapter-3.0_2.12-1.2.1-incubating.jar",
         2638338
        ],
        [
         "dbfs:/FileStore/jars/sedona/1.2.1-incubating/sedona-viz-3.0_2.12-1.2.1-incubating.jar",
         "sedona-viz-3.0_2.12-1.2.1-incubating.jar",
         429742
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "path",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "size",
         "type": "\"long\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%fs ls /FileStore/jars/sedona/1.2.1-incubating/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. Crear archivo de inicio "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "cfcf116b-5755-47d7-addd-49755de0575b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%sh \n",
    "\n",
    "# Create init script directory for Sedona\n",
    "mkdir -p /dbfs/FileStore/sedona/\n",
    "\n",
    "# Create init script\n",
    "cat > /dbfs/FileStore/sedona/sedona-init.sh <<'EOF'\n",
    "#!/bin/bash\n",
    "#\n",
    "# File: sedona-init.sh\n",
    "# Author: Erni Durdevic\n",
    "# Created: 2021-11-01\n",
    "# \n",
    "# On cluster startup, this script will copy the Sedona jars to the cluster's default jar directory.\n",
    "# In order to activate Sedona functions, remember to add to your spark configuration the Sedona extensions: \"spark.sql.extensions org.apache.sedona.viz.sql.SedonaVizExtensions,org.apache.sedona.sql.SedonaSqlExtensions\"\n",
    "\n",
    "cp /dbfs/FileStore/jars/sedona/1.2.1-incubating/*.jar /databricks/jars\n",
    "\n",
    "EOF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "e4835612-dc4c-4152-824e-a6790682c440",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "#### 5. Detener el cluster e agregar a los archivos de inicio la ruta del archivo anteriormente creado\n",
    "\n",
    "(dbfs:/FileStore/sedona/sedona-init.sh)\n",
    "\n",
    "#### 6. Reiniciar el clúster\n",
    "\n",
    "#### 7. instalar Apache Sedona como librería (apache-sedona)\n",
    "\n",
    "Para mayor información ver (https://sedona.apache.org/setup/install-python/#prepare-python-adapter-jar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "Install_Apache_sedona",
   "notebookOrigID": 4355108492540289,
   "widgets": {}
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}