create_project_functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4e259cbb-d8ab-4ae2-8d8c-fbeb6eeff49b",
   "metadata": {},
   "source": [
    "# Project creation notebook\n",
    "This notebook creates a project and sets all the different functions that will be used. It then saves the project to create a project.yaml file that can be used later to load the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "899ff237-5bb6-4ba5-a01e-aea754759636",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlrun\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ab604294-8367-4a0f-b370-339e81b73df3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2024-05-27 14:00:33,790 [info] Created and saved project: {'name': 'stress-project-amite', 'from_template': None, 'overwrite': False, 'context': './', 'save': True}\n",
      "> 2024-05-27 14:00:33,793 [info] Project created successfully: {'project_name': 'stress-project', 'stored_in_db': True}\n"
     ]
    }
   ],
   "source": [
    "project_name = \"stress-project\"\n",
    "project = mlrun.get_or_create_project(project_name, \"./\", user_project=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eba45321-177d-4f33-9345-3a6f0afa7897",
   "metadata": {},
   "source": [
    "# Setting the functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd1f241-a99f-4787-8f3e-e8ad2f538591",
   "metadata": {},
   "source": [
    "## Jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e248eb77-49d9-449e-9c7d-a2d3581251a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Names with underscore '_' are about to be deprecated, use dashes '-' instead. Replacing 'normal_job-min' underscores with dashes.\n"
     ]
    }
   ],
   "source": [
    "normal_job_sec = project.set_function(func=\"jobs_func.py\", name=\"normal-job-sec\", handler='func_sec', image=\"mlrun/mlrun\", kind=\"job\")\n",
    "normal_job_min = project.set_function(func=\"jobs_func.py\", name=\"normal_job-min\", handler='func_min', image=\"mlrun/mlrun\", kind=\"job\")\n",
    "normal_job_hour = project.set_function(func=\"jobs_func.py\", name=\"normal-job-hour\", handler='func_hour', image=\"mlrun/mlrun\", kind=\"job\")\n",
    "artifact_job = project.set_function(func=\"jobs_func.py\", name=\"artifact-job\", handler='func_artifact', image=\"mlrun/mlrun\", kind=\"job\")\n",
    "\n",
    "# Gets 2 parameters that define the dataframe size - \"num_rows\" (int) and \"num_columns\" (int)\n",
    "dataset_job = project.set_function(func=\"jobs_func.py\", name=\"dataset-job\", handler='func_dataset', image=\"mlrun/mlrun\", kind=\"job\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dafdbc3-6707-462d-9cb9-a1b8f619b33b",
   "metadata": {},
   "source": [
    "## Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "382004f9-803e-4fa4-a75c-1147fe58f28e",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_func = project.set_function(func=\"spark_jobs_func.py\", name=\"spark-job\", handler=\"spark_handler\", kind=\"spark\", image=\"mlrun/mlrun\").apply(mlrun.auto_mount())\n",
    "spark_func.with_executor_requests(cpu=\"1\",mem=\"1G\")\n",
    "spark_func.with_driver_requests(cpu=\"1\",mem=\"1G\")\n",
    "spark_func.with_driver_limits(cpu=\"1\")\n",
    "spark_func.with_executor_limits(cpu=\"1\")\n",
    "spark_func.with_igz_spark()\n",
    "spark_func.spec.image_pull_policy = \"Always\"\n",
    "spark_func.spec.replicas = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b4aac8a-208e-42bf-90a4-06369c0d7e35",
   "metadata": {},
   "source": [
    "## Dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7eed44c1-a033-4fc3-bdd9-d448f9a85e3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'db://stress-project-amite/dask-cluster'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "func = project.set_function(func='./dask_jobs_func.py',name='dask-func',image='mlrun/mlrun',kind='job',handler='train')\n",
    "dask_cluster = mlrun.new_function(\"dask-cluster\", kind='dask', image='mlrun/mlrun')\n",
    "dask_cluster.apply(mlrun.auto_mount())        # add volume mounts\n",
    "dask_cluster.spec.service_type = \"NodePort\"   # open interface to the dask UI dashboard\n",
    "dask_cluster.spec.replicas = 1             # define one container\n",
    "dask_cluster.set_env(\"MLRUN_DBPATH\",os.environ[\"MLRUN_DBPATH\"])\n",
    "dask_cluster.set_env(\"MLRUN_DEFAULT_PROJECT\",project.name)\n",
    "uri = dask_cluster.save()\n",
    "uri"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ce341e4-5688-4b40-afbe-83e39048c601",
   "metadata": {},
   "source": [
    "## Nuclio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f6e45ec7-1d53-41a6-9333-6938de26164e",
   "metadata": {},
   "outputs": [],
   "source": [
    "nuclio_func = project.set_function(\"nuclio_func.py\", name=\"nuclio-func\", handler=\"handler\", image=\"mlrun/mlrun\", kind=\"nuclio\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c76e7694-329c-404e-bb9f-5be4c1ab932a",
   "metadata": {},
   "source": [
    "## Log models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b4e6eba2-f2f5-4970-8d39-710b4cf436c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gets a parameter \"number_of_models\" (int) that define the number of models to create\n",
    "log_models_func = project.set_function(\"log_models.py\", name=\"log-models-job\", handler=\"log_n_models\", image=\"mlrun/mlrun\", kind=\"job\", requirements=[\"scikit-learn\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb7a48eb-e75e-455f-b75d-8c7773ff9442",
   "metadata": {},
   "source": [
    "## Serving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6dc92ccf-e784-47b7-b2c7-b51cc0b55dc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gets 2 parameters that define the number of servings and number of models in each serving - \"number_of_servings\" (int), \"number_of_models\" (int)\n",
    "serving_creator_func = project.set_function(\"serving_creator_func.py\", name=\"serving-creator-job\", handler=\"deploy_serving_functions\", image=\"mlrun/mlrun\", kind=\"job\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dea9132d-de72-473d-a2ed-cd37bc27c8d6",
   "metadata": {},
   "source": [
    "## Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a9e60317-db68-40e7-8492-5c600d509694",
   "metadata": {},
   "outputs": [],
   "source": [
    "project.set_workflow(name='workflow-func', workflow_path=\"workflow.py\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91b9cae8-26ef-460c-b4d2-7bcaf9545837",
   "metadata": {},
   "source": [
    "## Save project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "faf3f2a5-002e-4bbd-a4c4-3ee8d1b7d2a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.projects.project.MlrunProject at 0x7f8574931b80>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c8d2356-1bff-432c-839c-230bd1c556cb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlrun-base",
   "language": "python",
   "name": "conda-env-mlrun-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}