Skip to content

Commit d4e7795

Browse files
gilad-shahamgilad-shaham
authored
Merge pull request #354 from v3io/development
Remove magic commands and fix scripts
2 parents 3f1ff16 + 1848f5d commit d4e7795

13 files changed

+163
-412
lines changed

‎data-ingestion-and-preparation/dask-cluster.ipynb

+63-66
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
"name": "stdout",
4242
"output_type": "stream",
4343
"text": [
44-
"Both server & client are aligned (0.6.0rc13).\n"
44+
"Both server & client are aligned (0.7.1).\n"
4545
]
4646
}
4747
],
@@ -79,7 +79,7 @@
7979
"text": [
8080
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
8181
" Dload Upload Total Spent Left Speed\n",
82-
"100 84.9M 100 84.9M 0 0 8206k 0 0:00:10 0:00:10 --:--:-- 8639k\n"
82+
"100 84.9M 100 84.9M 0 0 21.7M 0 0:00:03 0:00:03 --:--:-- 21.7M\n"
8383
]
8484
}
8585
],
@@ -103,26 +103,6 @@
103103
"cell_type": "code",
104104
"execution_count": 5,
105105
"metadata": {},
106-
"outputs": [
107-
{
108-
"name": "stdout",
109-
"output_type": "stream",
110-
"text": [
111-
"%nuclio: setting kind to 'job'\n",
112-
"%nuclio: setting spec.image to 'mlrun/ml-models'\n"
113-
]
114-
}
115-
],
116-
"source": [
117-
"# nuclio: start-code\n",
118-
"%nuclio config kind = \"job\"\n",
119-
"%nuclio config spec.image = \"mlrun/ml-models\""
120-
]
121-
},
122-
{
123-
"cell_type": "code",
124-
"execution_count": 6,
125-
"metadata": {},
126106
"outputs": [],
127107
"source": [
128108
"from mlrun.execution import MLClientCtx\n",
@@ -157,7 +137,7 @@
157137
},
158138
{
159139
"cell_type": "code",
160-
"execution_count": 7,
140+
"execution_count": 6,
161141
"metadata": {},
162142
"outputs": [],
163143
"source": [
@@ -178,7 +158,7 @@
178158
},
179159
{
180160
"cell_type": "code",
181-
"execution_count": 8,
161+
"execution_count": 7,
182162
"metadata": {},
183163
"outputs": [],
184164
"source": [
@@ -194,9 +174,17 @@
194174
},
195175
{
196176
"cell_type": "code",
197-
"execution_count": 9,
177+
"execution_count": 8,
198178
"metadata": {},
199-
"outputs": [],
179+
"outputs": [
180+
{
181+
"name": "stdout",
182+
"output_type": "stream",
183+
"text": [
184+
"> 2021-10-12 09:43:54,745 [info] loaded project default from MLRun DB\n"
185+
]
186+
}
187+
],
200188
"source": [
201189
"import mlrun\n",
202190
"artifact_path = mlrun.set_environment(api_path = mlrun.mlconf.dbpath or 'http://mlrun-api:8080',\n",
@@ -220,7 +208,7 @@
220208
},
221209
{
222210
"cell_type": "code",
223-
"execution_count": 10,
211+
"execution_count": 9,
224212
"metadata": {},
225213
"outputs": [],
226214
"source": [
@@ -236,24 +224,16 @@
236224
},
237225
{
238226
"cell_type": "code",
239-
"execution_count": 11,
227+
"execution_count": 10,
240228
"metadata": {},
241-
"outputs": [
242-
{
243-
"name": "stdout",
244-
"output_type": "stream",
245-
"text": [
246-
"> 2021-02-04 19:07:34,933 [info] using in-cluster config.\n"
247-
]
248-
}
249-
],
229+
"outputs": [],
250230
"source": [
251231
"dsf = mlrun.new_function(\"dask_init\", kind='dask', image='mlrun/ml-models').apply(mlrun.mount_v3io())"
252232
]
253233
},
254234
{
255235
"cell_type": "code",
256-
"execution_count": 12,
236+
"execution_count": 11,
257237
"metadata": {},
258238
"outputs": [],
259239
"source": [
@@ -266,21 +246,21 @@
266246
},
267247
{
268248
"cell_type": "code",
269-
"execution_count": 13,
249+
"execution_count": 12,
270250
"metadata": {},
271251
"outputs": [
272252
{
273253
"name": "stdout",
274254
"output_type": "stream",
275255
"text": [
276-
"> 2021-02-04 19:07:35,158 [info] trying dask client at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n",
277-
"> 2021-02-04 19:07:35,234 [info] using remote dask scheduler (mlrun-dask-init-b7725876-7) at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n"
256+
"> 2021-10-12 09:44:06,095 [info] trying dask client at: tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786\n",
257+
"> 2021-10-12 09:44:06,116 [info] using remote dask scheduler (mlrun-dask-init-b76f44f5-0) at: tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786\n"
278258
]
279259
},
280260
{
281261
"data": {
282262
"text/html": [
283-
"<a href=\"http://default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053/status\" target=\"_blank\" >dashboard link: default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053</a>"
263+
"<a href=\"http://default-tenant.app.app-lab-eks-testing.iguazio-cd1.com:30990/status\" target=\"_blank\" >dashboard link: default-tenant.app.app-lab-eks-testing.iguazio-cd1.com:30990</a>"
284264
],
285265
"text/plain": [
286266
"<IPython.core.display.HTML object>"
@@ -297,26 +277,26 @@
297277
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
298278
"<h3 style=\"text-align: left;\">Client</h3>\n",
299279
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
300-
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</li>\n",
301-
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-b7725876-7.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-b7725876-7.default-tenant:8787/status</a></li>\n",
280+
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786</li>\n",
281+
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-b76f44f5-0.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-b76f44f5-0.default-tenant:8787/status</a></li>\n",
302282
"</ul>\n",
303283
"</td>\n",
304284
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
305285
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
306286
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
307287
" <li><b>Workers: </b>2</li>\n",
308288
" <li><b>Cores: </b>2</li>\n",
309-
" <li><b>Memory: </b>8.31 GB</li>\n",
289+
" <li><b>Memory: </b>8.34 GB</li>\n",
310290
"</ul>\n",
311291
"</td>\n",
312292
"</tr>\n",
313293
"</table>"
314294
],
315295
"text/plain": [
316-
"<Client: 'tcp://10.200.0.59:8786' processes=2 threads=2, memory=8.31 GB>"
296+
"<Client: 'tcp://172.31.13.112:8786' processes=2 threads=2, memory=8.34 GB>"
317297
]
318298
},
319-
"execution_count": 13,
299+
"execution_count": 12,
320300
"metadata": {},
321301
"output_type": "execute_result"
322302
}
@@ -335,7 +315,7 @@
335315
},
336316
{
337317
"cell_type": "code",
338-
"execution_count": 14,
318+
"execution_count": 13,
339319
"metadata": {},
340320
"outputs": [],
341321
"source": [
@@ -360,16 +340,16 @@
360340
},
361341
{
362342
"cell_type": "code",
363-
"execution_count": 15,
343+
"execution_count": 14,
364344
"metadata": {},
365345
"outputs": [
366346
{
367347
"name": "stdout",
368348
"output_type": "stream",
369349
"text": [
370-
"> 2021-02-04 19:07:35,288 [info] starting run test-dask-test_dask uid=4973d57c968747e491d6fcaa401968e0 DB=http://mlrun-api:8080\n",
371-
"> 2021-02-04 19:07:35,769 [info] Job is running in the background, pod: test-dask-test-dask-4m29c\n",
372-
"> 2021-02-04 19:07:56,270 [info] run executed, status=completed\n",
350+
"> 2021-10-12 09:44:06,154 [info] starting run test-dask-test_dask uid=3d3ec4b558a140b5849700dc5d77c97c DB=http://mlrun-api:8080\n",
351+
"> 2021-10-12 09:44:06,317 [info] Job is running in the background, pod: test-dask-test-dask-w28ff\n",
352+
"> 2021-10-12 09:44:20,436 [info] run executed, status=completed\n",
373353
"final state: completed\n"
374354
]
375355
},
@@ -544,26 +524,26 @@
544524
" <tbody>\n",
545525
" <tr>\n",
546526
" <td>default</td>\n",
547-
" <td><div title=\"4973d57c968747e491d6fcaa401968e0\"><a href=\"https://dashboard.default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com/mlprojects/default/jobs/monitor/4973d57c968747e491d6fcaa401968e0/info\" target=\"_blank\" >...401968e0</a></div></td>\n",
527+
" <td><div title=\"3d3ec4b558a140b5849700dc5d77c97c\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks-testing.iguazio-cd1.com/mlprojects/default/jobs/monitor/3d3ec4b558a140b5849700dc5d77c97c/overview\" target=\"_blank\" >...5d77c97c</a></div></td>\n",
548528
" <td>0</td>\n",
549-
" <td>Feb 04 19:07:42</td>\n",
529+
" <td>Oct 12 09:44:11</td>\n",
550530
" <td>completed</td>\n",
551531
" <td>test-dask-test_dask</td>\n",
552-
" <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=test-dask-test-dask-4m29c</div></td>\n",
553-
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultf45735f7\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
554-
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</div></td>\n",
532+
" <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">host=test-dask-test-dask-w28ff</div></td>\n",
533+
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result46fb98e5\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
534+
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786</div></td>\n",
555535
" <td></td>\n",
556536
" <td></td>\n",
557537
" </tr>\n",
558538
" </tbody>\n",
559539
"</table>\n",
560540
"</div></div>\n",
561-
" <div id=\"resultf45735f7-pane\" class=\"right-pane block hidden\">\n",
541+
" <div id=\"result46fb98e5-pane\" class=\"right-pane block hidden\">\n",
562542
" <div class=\"pane-header\">\n",
563-
" <span id=\"resultf45735f7-title\" class=\"pane-header-title\">Title</span>\n",
564-
" <span onclick=\"closePanel(this)\" paneName=\"resultf45735f7\" class=\"close clickable\">&times;</span>\n",
543+
" <span id=\"result46fb98e5-title\" class=\"pane-header-title\">Title</span>\n",
544+
" <span onclick=\"closePanel(this)\" paneName=\"result46fb98e5\" class=\"close clickable\">&times;</span>\n",
565545
" </div>\n",
566-
" <iframe class=\"fileview\" id=\"resultf45735f7-body\"></iframe>\n",
546+
" <iframe class=\"fileview\" id=\"result46fb98e5-body\"></iframe>\n",
567547
" </div>\n",
568548
"</div>\n"
569549
],
@@ -578,18 +558,35 @@
578558
"name": "stdout",
579559
"output_type": "stream",
580560
"text": [
581-
"to track results use .show() or .logs() or in CLI: \n",
582-
"!mlrun get run 4973d57c968747e491d6fcaa401968e0 --project default , !mlrun logs 4973d57c968747e491d6fcaa401968e0 --project default\n",
583-
"> 2021-02-04 19:08:05,372 [info] run executed, status=completed\n"
561+
"\n"
562+
]
563+
},
564+
{
565+
"data": {
566+
"text/html": [
567+
"<b> > to track results use the .show() or .logs() methods or <a href=\"https://dashboard.default-tenant.app.app-lab-eks-testing.iguazio-cd1.com/mlprojects/default/jobs/monitor/3d3ec4b558a140b5849700dc5d77c97c/overview\" target=\"_blank\">click here</a> to open in UI</b>"
568+
],
569+
"text/plain": [
570+
"<IPython.core.display.HTML object>"
571+
]
572+
},
573+
"metadata": {},
574+
"output_type": "display_data"
575+
},
576+
{
577+
"name": "stdout",
578+
"output_type": "stream",
579+
"text": [
580+
"> 2021-10-12 09:44:25,557 [info] run executed, status=completed\n"
584581
]
585582
},
586583
{
587584
"data": {
588585
"text/plain": [
589-
"<mlrun.model.RunObject at 0x7efe58c67c50>"
586+
"<mlrun.model.RunObject at 0x7fe5ec4a1d50>"
590587
]
591588
},
592-
"execution_count": 15,
589+
"execution_count": 14,
593590
"metadata": {},
594591
"output_type": "execute_result"
595592
}

0 commit comments

Comments
 (0)