Skip to content
This repository was archived by the owner on Apr 14, 2023. It is now read-only.

Commit d9347d1

Browse files
tkakantousistkakantousis
committed
Merge branch 'cherry_pick_0.4' into release-0.4
# Conflicts: # flink/pom.xml # hive/pom.xml # pom.xml # spark/pom.xml
2 parents d5b25e8 + 60d1aa3 commit d9347d1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1563
-635
lines changed

‎tensorflow/notebooks/PyTorch/mnist.ipynb‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"Pytorch example"
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": 1,
13+
"metadata": {
14+
"collapsed": false,
15+
"deletable": true,
16+
"editable": true
17+
},
18+
"outputs": [
19+
{
20+
"name": "stdout",
21+
"output_type": "stream",
22+
"text": [
23+
"Starting Spark application\n"
24+
]
25+
},
26+
{
27+
"data": {
28+
"text/html": [
29+
"<table>\n",
30+
"<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>7717</td><td>application_1513605045578_5456</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://hadoop30:8088/proxy/application_1513605045578_5456/\">Link</a></td><td><a target=\"_blank\" href=\"http://hadoop17:8042/node/containerlogs/container_e28_1513605045578_5456_01_000001/copystufftest__robin_er\">Link</a></td><td>✔</td></tr></table>"
31+
],
32+
"text/plain": [
33+
"<IPython.core.display.HTML object>"
34+
]
35+
},
36+
"metadata": {},
37+
"output_type": "display_data"
38+
},
39+
{
40+
"name": "stdout",
41+
"output_type": "stream",
42+
"text": [
43+
"SparkSession available as 'spark'.\n"
44+
]
45+
}
46+
],
47+
"source": [
48+
"def wrapper():\n",
49+
" import argparse\n",
50+
" import torch\n",
51+
" import torch.nn as nn\n",
52+
" import torch.nn.functional as F\n",
53+
" import torch.optim as optim\n",
54+
" from torchvision import datasets, transforms\n",
55+
" from torch.autograd import Variable\n",
56+
"\n",
57+
" # Training settings\n",
58+
" parser = argparse.ArgumentParser(description='PyTorch MNIST Example')\n",
59+
" parser.add_argument('--batch-size', type=int, default=64, metavar='N',\n",
60+
" help='input batch size for training (default: 64)')\n",
61+
" parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',\n",
62+
" help='input batch size for testing (default: 1000)')\n",
63+
" parser.add_argument('--epochs', type=int, default=10, metavar='N',\n",
64+
" help='number of epochs to train (default: 10)')\n",
65+
" parser.add_argument('--lr', type=float, default=0.01, metavar='LR',\n",
66+
" help='learning rate (default: 0.01)')\n",
67+
" parser.add_argument('--momentum', type=float, default=0.5, metavar='M',\n",
68+
" help='SGD momentum (default: 0.5)')\n",
69+
" parser.add_argument('--no-cuda', action='store_true', default=False,\n",
70+
" help='disables CUDA training')\n",
71+
" parser.add_argument('--seed', type=int, default=1, metavar='S',\n",
72+
" help='random seed (default: 1)')\n",
73+
" parser.add_argument('--log-interval', type=int, default=10, metavar='N',\n",
74+
" help='how many batches to wait before logging training status')\n",
75+
" args = parser.parse_args()\n",
76+
" args.cuda = not args.no_cuda and torch.cuda.is_available()\n",
77+
"\n",
78+
" torch.manual_seed(args.seed)\n",
79+
" if args.cuda:\n",
80+
" torch.cuda.manual_seed(args.seed)\n",
81+
"\n",
82+
"\n",
83+
" kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}\n",
84+
" train_loader = torch.utils.data.DataLoader(\n",
85+
" datasets.MNIST('../data', train=True, download=True,\n",
86+
" transform=transforms.Compose([\n",
87+
" transforms.ToTensor(),\n",
88+
" transforms.Normalize((0.1307,), (0.3081,))\n",
89+
" ])),\n",
90+
" batch_size=args.batch_size, shuffle=True, **kwargs)\n",
91+
" test_loader = torch.utils.data.DataLoader(\n",
92+
" datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
93+
" transforms.ToTensor(),\n",
94+
" transforms.Normalize((0.1307,), (0.3081,))\n",
95+
" ])),\n",
96+
" batch_size=args.test_batch_size, shuffle=True, **kwargs)\n",
97+
"\n",
98+
"\n",
99+
" class Net(nn.Module):\n",
100+
" def __init__(self):\n",
101+
" super(Net, self).__init__()\n",
102+
" self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
103+
" self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
104+
" self.conv2_drop = nn.Dropout2d()\n",
105+
" self.fc1 = nn.Linear(320, 50)\n",
106+
" self.fc2 = nn.Linear(50, 10)\n",
107+
"\n",
108+
" def forward(self, x):\n",
109+
" x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
110+
" x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
111+
" x = x.view(-1, 320)\n",
112+
" x = F.relu(self.fc1(x))\n",
113+
" x = F.dropout(x, training=self.training)\n",
114+
" x = self.fc2(x)\n",
115+
" return F.log_softmax(x)\n",
116+
"\n",
117+
" model = Net()\n",
118+
" if args.cuda:\n",
119+
" model.cuda()\n",
120+
"\n",
121+
" optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)\n",
122+
"\n",
123+
" def train(epoch):\n",
124+
" model.train()\n",
125+
" for batch_idx, (data, target) in enumerate(train_loader):\n",
126+
" if args.cuda:\n",
127+
" data, target = data.cuda(), target.cuda()\n",
128+
" data, target = Variable(data), Variable(target)\n",
129+
" optimizer.zero_grad()\n",
130+
" output = model(data)\n",
131+
" loss = F.nll_loss(output, target)\n",
132+
" loss.backward()\n",
133+
" optimizer.step()\n",
134+
" if batch_idx % args.log_interval == 0:\n",
135+
" print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
136+
" epoch, batch_idx * len(data), len(train_loader.dataset),\n",
137+
" 100. * batch_idx / len(train_loader), loss.data[0]))\n",
138+
"\n",
139+
" def test():\n",
140+
" model.eval()\n",
141+
" test_loss = 0\n",
142+
" correct = 0\n",
143+
" for data, target in test_loader:\n",
144+
" if args.cuda:\n",
145+
" data, target = data.cuda(), target.cuda()\n",
146+
" data, target = Variable(data, volatile=True), Variable(target)\n",
147+
" output = model(data)\n",
148+
" test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss\n",
149+
" pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
150+
" correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
151+
"\n",
152+
" test_loss /= len(test_loader.dataset)\n",
153+
" print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
154+
" test_loss, correct, len(test_loader.dataset),\n",
155+
" 100. * correct / len(test_loader.dataset)))\n",
156+
"\n",
157+
"\n",
158+
" for epoch in range(1, args.epochs + 1):\n",
159+
" train(epoch)\n",
160+
" test()"
161+
]
162+
},
163+
{
164+
"cell_type": "code",
165+
"execution_count": null,
166+
"metadata": {
167+
"collapsed": false,
168+
"deletable": true,
169+
"editable": true
170+
},
171+
"outputs": [],
172+
"source": [
173+
"from hops import experiment\n",
174+
"experiment.launch(spark, wrapper)"
175+
]
176+
},
177+
{
178+
"cell_type": "code",
179+
"execution_count": null,
180+
"metadata": {
181+
"collapsed": true,
182+
"deletable": true,
183+
"editable": true
184+
},
185+
"outputs": [],
186+
"source": []
187+
}
188+
],
189+
"metadata": {
190+
"kernelspec": {
191+
"display_name": "PySpark",
192+
"language": "",
193+
"name": "pysparkkernel"
194+
},
195+
"language_info": {
196+
"codemirror_mode": {
197+
"name": "python",
198+
"version": 2
199+
},
200+
"mimetype": "text/x-python",
201+
"name": "pyspark",
202+
"pygments_lexer": "python2"
203+
}
204+
},
205+
"nbformat": 4,
206+
"nbformat_minor": 2
207+
}

‎tensorflow/notebooks/Serving/serving_client.ipynb‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": null,
6+
"metadata": {
7+
"scrolled": true
8+
},
9+
"outputs": [],
10+
"source": [
11+
"from __future__ import print_function\n",
12+
"\n",
13+
"import sys\n",
14+
"import threading\n",
15+
"import os\n",
16+
"\n",
17+
"from grpc.beta import implementations\n",
18+
"import numpy\n",
19+
"import tensorflow as tf\n",
20+
"\n",
21+
"from tensorflow_serving.apis import predict_pb2\n",
22+
"from tensorflow_serving.apis import prediction_service_pb2\n",
23+
"from tensorflow.examples.tutorials.mnist import input_data as mnist_input_data\n",
24+
"\n",
25+
"from hops import serving\n",
26+
"from hops import hdfs\n",
27+
"\n",
28+
"concurrency=1\n",
29+
"num_tests=100\n",
30+
"work_dir=os.getcwd()\n",
31+
"server=\"host:ip\"\n",
32+
"\n",
33+
"\n",
34+
"class _ResultCounter(object):\n",
35+
" \"\"\"Counter for the prediction results.\"\"\"\n",
36+
"\n",
37+
" def __init__(self, num_tests, concurrency):\n",
38+
" self._num_tests = num_tests\n",
39+
" self._concurrency = concurrency\n",
40+
" self._error = 0\n",
41+
" self._done = 0\n",
42+
" self._active = 0\n",
43+
" self._condition = threading.Condition()\n",
44+
"\n",
45+
" def inc_error(self):\n",
46+
" with self._condition:\n",
47+
" self._error += 1\n",
48+
"\n",
49+
" def inc_done(self):\n",
50+
" with self._condition:\n",
51+
" self._done += 1\n",
52+
" self._condition.notify()\n",
53+
"\n",
54+
" def dec_active(self):\n",
55+
" with self._condition:\n",
56+
" self._active -= 1\n",
57+
" self._condition.notify()\n",
58+
"\n",
59+
" def get_error_rate(self):\n",
60+
" with self._condition:\n",
61+
" while self._done != self._num_tests:\n",
62+
" self._condition.wait()\n",
63+
" return self._error / float(self._num_tests)\n",
64+
"\n",
65+
" def throttle(self):\n",
66+
" with self._condition:\n",
67+
" while self._active == self._concurrency:\n",
68+
" self._condition.wait()\n",
69+
" self._active += 1\n",
70+
"\n",
71+
"\n",
72+
"def _create_rpc_callback(label, result_counter):\n",
73+
" \"\"\"Creates RPC callback function.\n",
74+
" Args:\n",
75+
" label: The correct label for the predicted example.\n",
76+
" result_counter: Counter for the prediction result.\n",
77+
" Returns:\n",
78+
" The callback function.\n",
79+
" \"\"\"\n",
80+
" def _callback(result_future):\n",
81+
" \"\"\"Callback function.\n",
82+
" Calculates the statistics for the prediction result.\n",
83+
" Args:\n",
84+
" result_future: Result future of the RPC.\n",
85+
" \"\"\"\n",
86+
" exception = result_future.exception()\n",
87+
" if exception:\n",
88+
" result_counter.inc_error()\n",
89+
" print(exception)\n",
90+
" else:\n",
91+
" sys.stdout.write('.')\n",
92+
" sys.stdout.flush()\n",
93+
" response = numpy.array(\n",
94+
" result_future.result().outputs['scores'].float_val)\n",
95+
" prediction = numpy.argmax(response)\n",
96+
" if label != prediction:\n",
97+
" result_counter.inc_error()\n",
98+
" result_counter.inc_done()\n",
99+
" result_counter.dec_active()\n",
100+
" return _callback\n",
101+
"\n",
102+
"\n",
103+
"def do_inference(hostport, work_dir, concurrency, num_tests):\n",
104+
" \"\"\"Tests PredictionService with concurrent requests.\n",
105+
" Args:\n",
106+
" hostport: Host:port address of the PredictionService.\n",
107+
" work_dir: The full path of working directory for test data set.\n",
108+
" concurrency: Maximum number of concurrent requests.\n",
109+
" num_tests: Number of test images to use.\n",
110+
" Returns:\n",
111+
" The classification error rate.\n",
112+
" Raises:\n",
113+
" IOError: An error occurred processing test data set.\n",
114+
" \"\"\"\n",
115+
" test_data_set = mnist_input_data.read_data_sets(work_dir).test\n",
116+
" host, port = hostport.split(':')\n",
117+
" channel = implementations.insecure_channel(host, int(port))\n",
118+
" stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)\n",
119+
" result_counter = _ResultCounter(num_tests, concurrency)\n",
120+
" for _ in range(num_tests):\n",
121+
" request = predict_pb2.PredictRequest()\n",
122+
" request.model_spec.name = 'mnist'\n",
123+
" request.model_spec.signature_name = 'predict_images'\n",
124+
" image, label = test_data_set.next_batch(1)\n",
125+
" request.inputs['images'].CopyFrom(\n",
126+
" tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size]))\n",
127+
" result_counter.throttle()\n",
128+
" result_future = stub.Predict.future(request, 5.0) # 5 seconds\n",
129+
" result_future.add_done_callback(\n",
130+
" _create_rpc_callback(label[0], result_counter))\n",
131+
" return result_counter.get_error_rate()\n",
132+
"\n",
133+
"\n",
134+
"\n",
135+
"error_rate = do_inference(server, work_dir, concurrency, num_tests)\n",
136+
"print('\\nAccuracy : %s%%' % (100 - (error_rate * 100)))\n"
137+
]
138+
}
139+
],
140+
"metadata": {
141+
"kernelspec": {
142+
"display_name": "PySpark",
143+
"language": "",
144+
"name": "pysparkkernel"
145+
},
146+
"language_info": {
147+
"codemirror_mode": {
148+
"name": "python",
149+
"version": 2
150+
},
151+
"mimetype": "text/x-python",
152+
"name": "pyspark",
153+
"pygments_lexer": "python2"
154+
}
155+
},
156+
"nbformat": 4,
157+
"nbformat_minor": 2
158+
}

0 commit comments

Comments
 (0)