chore: Reduce package deps, improve install and import time (wandb#1144)

* Simplify intro again

* Start of remove package deps refactor

* Down to 204 failures

* Fix more tests

* Fix more tests

* More test fixes

* Fix test

* Fixes

* Add import safety check

* Refactor: split arrow from ops_arrow

* Fix unit tests

* Fix more tests.

* Remove unneeded import

* Fix tests.

* fix node_ref test

* Fix lint

* Disable test

* Disable test

* Fix package

* Remove extra file

* Fix notebook test

* make panels render in notebooks

* Fix notebook

* Fix a few notebooks

* Fix another notebook

* Fix synced axes notebook

* Fix broken panel rendering and add test back

* Fix some notebook tests, remove others

* Notebook auto-lint

* Simplify requirements for weaveflow

* Fix WB_API notebook

* Fix notebook

* Fix scatter notebook

* Update other requirements to point to engine

* Fix confusion matrix test

* Fix requirements structure

* Fix autoboards and notebook test

* Clean up

* Empty-Commit

Author: shawnlewis

.pre-commit-config.yaml

@@ -16,10 +16,11 @@ repos:
       - id: mypy
         additional_dependencies: [types-all, wandb>=0.15.5]
         exclude: .*pyi$
-  - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.341
-    hooks:
-    - id: pyright
+  # Turn pyright back off, duplicative of mypy
+  # - repo: https://github.com/RobertCraigie/pyright-python
+  #   rev: v1.1.341
+  #   hooks:
+  #   - id: pyright
   - repo: local
     hooks:
       - id: jupyter-nb-clear-output

Dockerfile

@@ -42,7 +42,7 @@ ENV PORT 9239
 WORKDIR /weave
 ADD . .
 
-RUN pip install -r requirements.txt
+RUN pip install -r requirements.engine.txt
 RUN mkdir /local-artifacts
 
 EXPOSE 9239

MANIFEST.in

@@ -1,5 +1,6 @@
 include requirements.*
 include wb_schema.gql
+graft weave/arrow
 graft weave/frontend
 graft weave/language_features
 graft weave/ops_primitives

examples/experimental/01 Tutorial - Operations.ipynb

@@ -4,7 +4,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "69440036",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:14.261867Z",
+     "iopub.status.busy": "2024-01-27T16:44:14.261748Z",
+     "iopub.status.idle": "2024-01-27T16:44:15.442511Z",
+     "shell.execute_reply": "2024-01-27T16:44:15.442129Z"
+    }
+   },
    "outputs": [],
    "source": [
     "import weave\n",
@@ -16,7 +23,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "a7f7eda3",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:15.444502Z",
+     "iopub.status.busy": "2024-01-27T16:44:15.444298Z",
+     "iopub.status.idle": "2024-01-27T16:44:15.448977Z",
+     "shell.execute_reply": "2024-01-27T16:44:15.448714Z"
+    }
+   },
    "outputs": [],
    "source": [
     "x = 7\n",
@@ -30,7 +44,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "e36d5f49",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:15.450446Z",
+     "iopub.status.busy": "2024-01-27T16:44:15.450342Z",
+     "iopub.status.idle": "2024-01-27T16:44:15.452299Z",
+     "shell.execute_reply": "2024-01-27T16:44:15.452049Z"
+    }
+   },
    "outputs": [],
    "source": [
     "print(add_to_closed_var.resolve_fn.__closure__)"
@@ -40,7 +61,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "287a502c",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:15.453690Z",
+     "iopub.status.busy": "2024-01-27T16:44:15.453585Z",
+     "iopub.status.idle": "2024-01-27T16:44:19.844023Z",
+     "shell.execute_reply": "2024-01-27T16:44:19.843685Z"
+    }
+   },
    "outputs": [],
    "source": [
     "print(weave.use(add_to_closed_var(9)))"
@@ -50,7 +78,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "64f7c176",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:19.845702Z",
+     "iopub.status.busy": "2024-01-27T16:44:19.845472Z",
+     "iopub.status.idle": "2024-01-27T16:44:19.849105Z",
+     "shell.execute_reply": "2024-01-27T16:44:19.848830Z"
+    }
+   },
    "outputs": [],
    "source": [
     "outer_var = 1\n",
@@ -69,7 +104,14 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "4f01dcf4",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-01-27T16:44:19.850537Z",
+     "iopub.status.busy": "2024-01-27T16:44:19.850451Z",
+     "iopub.status.idle": "2024-01-27T16:44:19.854527Z",
+     "shell.execute_reply": "2024-01-27T16:44:19.854126Z"
+    }
+   },
    "outputs": [],
    "source": [
     "outer_var = 1\n",

examples/experimental/Closures.ipynb

@@ -22,7 +22,7 @@
     "# Weave package now defaults to eager mode, but lazy mode required for this example notebook for now.\n",
     "weave.use_lazy_execution()\n",
     "from weave import syndata_mon\n",
-    "from weave.ops_arrow.list_ import dataframe_to_arrow\n",
+    "from weave.arrow.list_ import dataframe_to_arrow\n",
     "from weave.ecosystem import wandb"
    ]
   },

examples/experimental/Monitor.ipynb

@@ -18,7 +18,7 @@
    "outputs": [],
    "source": [
     "import weave\n",
-    "weave.use_frontend_devmode()\n",
+    "import weave.panels\n",
     "weave.use_lazy_execution()\n",
     "from weave import syndata_mon"
    ]

examples/experimental/MonitorPanelPlot.ipynb

@@ -13,6 +13,7 @@
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
     "import random\n",
+    "from weave import ops\n",
     "\n",
     "def plt_image():\n",
     "    \"\"\"Return current matplotlib figure as PIL Image\"\"\"\n",
@@ -117,7 +118,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,

examples/experimental/image_gen_ops.ipynb

@@ -57,7 +57,7 @@
     "plot = weave.panels.Plot(points,\n",
     "                         x=lambda row: row['x'],\n",
     "                         y=lambda row: row['y'])\n",
-    "weave.show(plot)\n",
+    "plot\n",
     "\n",
     "# NOTE! There is a UI bug here. You need to click the gear icon and\n",
     "#     then \"Reset & Automate Plot\" -> \"OK\" to make it render correctly"

examples/experimental/inference_logs.ipynb

@@ -8,6 +8,7 @@
    "outputs": [],
    "source": [
     "import weave\n",
+    "import weave.panels\n",
     "# Weave package now defaults to eager mode, but lazy mode required for this example notebook for now.\n",
     "weave.use_lazy_execution()"
    ]

examples/experimental/object_version_compare.ipynb

@@ -76,7 +76,8 @@
    "outputs": [],
    "source": [
     "import weave\n",
-    "weave.show(iris_df)"
+    "from weave.show import show\n",
+    "show(iris_df)"
    ]
   },
   {
@@ -129,7 +130,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "weave.show(iris_df)"
+    "show(iris_df)"
    ]
   },
   {
@@ -205,7 +206,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "weave.show(iris_df)"
+    "show(iris_df)"
    ]
   },
   {
@@ -251,7 +252,7 @@
     "    imarray = numpy.random.rand(100,100,3) * 255\n",
     "    return Image.fromarray(imarray.astype('uint8')).convert('RGBA')\n",
     "\n",
-    "weave.show([{'image_col': random_image(), 'text_col': 'Hello'}, \n",
+    "show([{'image_col': random_image(), 'text_col': 'Hello'}, \n",
     "            {'image_col': random_image(), 'text_col': 'World'}]*1000)"
    ]
   },