From bbdf6faa001747da50688e1a2ddc5e5e20ec97b9 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 12:44:28 -0400 Subject: [PATCH 01/21] Update for Python3 --- digits/config/__init__.py | 2 ++ digits/config/caffe.py | 49 +++++++++++++++++------------------ digits/config/gpu_list.py | 2 +- digits/config/jobs_dir.py | 4 +-- digits/config/log_file.py | 4 +-- digits/config/store_option.py | 4 +-- digits/config/tensorflow.py | 2 +- 7 files changed, 34 insertions(+), 33 deletions(-) diff --git a/digits/config/__init__.py b/digits/config/__init__.py index 965fb19b7..953e6b8a3 100644 --- a/digits/config/__init__.py +++ b/digits/config/__init__.py @@ -16,6 +16,8 @@ url_prefix, ) +if option_list['caffe']['loaded'] is False and option_list['tensorflow']['enabled']: + option_list['caffe']['multi_gpu'] = True def config_value(option): """ diff --git a/digits/config/caffe.py b/digits/config/caffe.py index db8aedc94..160cf6fc1 100644 --- a/digits/config/caffe.py +++ b/digits/config/caffe.py @@ -37,10 +37,7 @@ def load_from_envvar(envvar): import_pycaffe(python_dir) version, flavor = get_version_and_flavor(executable) except: - print ('"%s" from %s does not point to a valid installation of Caffe.' - % (value, envvar)) - print 'Use the envvar CAFFE_ROOT to indicate a valid installation.' - raise + raise ('"%s" from %s does not point to a valid installation of Caffe. \nUse the envvar CAFFE_ROOT to indicate a valid installation.'% (value, envvar)) return executable, version, flavor @@ -57,9 +54,7 @@ def load_from_path(): import_pycaffe() version, flavor = get_version_and_flavor(executable) except: - print 'A valid Caffe installation was not found on your system.' - print 'Use the envvar CAFFE_ROOT to indicate a valid installation.' - raise + raise ('A valid Caffe installation was not found on your system. \nUse the envvar CAFFE_ROOT to indicate a valid installation.') return executable, version, flavor @@ -125,8 +120,7 @@ def import_pycaffe(dirname=None): try: import caffe except ImportError: - print 'Did you forget to "make pycaffe"?' - raise + raise ('Did you forget to "make pycaffe"?') # Strange issue with protocol buffers and pickle - see issue #32 sys.path.insert(0, os.path.join( @@ -181,7 +175,7 @@ def get_version_from_cmdline(executable): command = [executable, '-version'] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.wait(): - print p.stderr.read().strip() + print(p.stderr.read().strip()) raise RuntimeError('"%s" returned error code %s' % (command, p.returncode)) pattern = 'version' @@ -195,7 +189,7 @@ def get_version_from_soname(executable): command = ['ldd', executable] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if p.wait(): - print p.stderr.read().strip() + print(p.stderr.read().strip()) raise RuntimeError('"%s" returned error code %s' % (command, p.returncode)) # Search output for caffe library @@ -222,17 +216,22 @@ def get_version_from_soname(executable): return None -if 'CAFFE_ROOT' in os.environ: - executable, version, flavor = load_from_envvar('CAFFE_ROOT') -elif 'CAFFE_HOME' in os.environ: - executable, version, flavor = load_from_envvar('CAFFE_HOME') -else: - executable, version, flavor = load_from_path() - -option_list['caffe'] = { - 'executable': executable, - 'version': version, - 'flavor': flavor, - 'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0, 12)), - 'cuda_enabled': (len(device_query.get_devices()) > 0), -} +try: + if 'CAFFE_ROOT' in os.environ: + executable, version, flavor = load_from_envvar('CAFFE_ROOT') + elif 'CAFFE_HOME' in os.environ: + executable, version, flavor = load_from_envvar('CAFFE_HOME') + else: + executable, version, flavor = load_from_path() + option_list['caffe'] = { + 'loaded': True, + 'executable': executable, + 'version': version, + 'flavor': flavor, + 'multi_gpu': (flavor == 'BVLC' or parse_version(version) >= parse_version(0, 12)), + 'cuda_enabled': (len(device_query.get_devices()) > 0), + } +except (Exception, ImportError, RuntimeError) as e: + print("Caffe support disabled.") + # print("Reason: {}".format(e.message)) + option_list['caffe'] = {'loaded': False, 'multi_gpu': False, 'cuda_enabled': False} diff --git a/digits/config/gpu_list.py b/digits/config/gpu_list.py index 32c9b6e41..1a30a5379 100644 --- a/digits/config/gpu_list.py +++ b/digits/config/gpu_list.py @@ -5,4 +5,4 @@ import digits.device_query -option_list['gpu_list'] = ','.join([str(x) for x in xrange(len(digits.device_query.get_devices()))]) +option_list['gpu_list'] = ','.join([str(x) for x in range(len(digits.device_query.get_devices()))]) diff --git a/digits/config/jobs_dir.py b/digits/config/jobs_dir.py index e28d5ea68..103ce5a24 100644 --- a/digits/config/jobs_dir.py +++ b/digits/config/jobs_dir.py @@ -26,8 +26,8 @@ if not os.path.exists(value): os.makedirs(value) except: - print '"%s" is not a valid value for jobs_dir.' % value - print 'Set the envvar DIGITS_JOBS_DIR to fix your configuration.' + print('"%s" is not a valid value for jobs_dir.' % value) + print('Set the envvar DIGITS_JOBS_DIR to fix your configuration.') raise diff --git a/digits/config/log_file.py b/digits/config/log_file.py index fe469052a..962975c9e 100644 --- a/digits/config/log_file.py +++ b/digits/config/log_file.py @@ -32,8 +32,8 @@ def load_logfile_filename(): pass except: if throw_error: - print '"%s" is not a valid value for logfile_filename.' % filename - print 'Set the envvar DIGITS_LOGFILE_FILENAME to fix your configuration.' + print('"%s" is not a valid value for logfile_filename.' % filename) + print('Set the envvar DIGITS_LOGFILE_FILENAME to fix your configuration.') raise else: filename = None diff --git a/digits/config/store_option.py b/digits/config/store_option.py index 3721a178c..a23f680fa 100644 --- a/digits/config/store_option.py +++ b/digits/config/store_option.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import os -from urlparse import urlparse +from urllib import parse from . import option_list @@ -14,7 +14,7 @@ def validate(value): if isinstance(value, str): url_list = value.split(',') for url in url_list: - if url is not None and urlparse(url).scheme != "" and not os.path.exists(url): + if url is not None and parse.urlparse(url).scheme != "" and not os.path.exists(url): valid_url_list.append(url) else: raise ValueError('"%s" is not a valid URL' % url) diff --git a/digits/config/tensorflow.py b/digits/config/tensorflow.py index b4eba67f3..0304bb689 100644 --- a/digits/config/tensorflow.py +++ b/digits/config/tensorflow.py @@ -10,7 +10,7 @@ def test_tf_import(): try: import tensorflow # noqa return True - except (ImportError, TypeError): + except ImportError: return False tf_enabled = test_tf_import() From 4de376d0bb4190652f319590ba8d8b1bbfed06bf Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 12:45:58 -0400 Subject: [PATCH 02/21] Update for Python3 --- digits/dataset/generic/test_views.py | 24 ++++++++++++------------ digits/dataset/generic/views.py | 14 ++++++++------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/digits/dataset/generic/test_views.py b/digits/dataset/generic/test_views.py index 12da7d633..526648622 100644 --- a/digits/dataset/generic/test_views.py +++ b/digits/dataset/generic/test_views.py @@ -88,16 +88,16 @@ def create_dataset(cls, **kwargs): if rv.status_code != 200: raise RuntimeError( 'Dataset creation failed with %s' % rv.status_code) - return json.loads(rv.data)['id'] + return json.loads(rv.get_data(as_text=True))['id'] # expect a redirect if not 300 <= rv.status_code <= 310: s = BeautifulSoup(rv.data, 'html.parser') div = s.select('div.alert-danger') if div: - print div[0] + print(div[0]) else: - print rv.data + print(rv.get_data(as_text=True)) raise RuntimeError( 'Failed to create dataset - status %s' % rv.status_code) @@ -112,7 +112,7 @@ def create_dataset(cls, **kwargs): def get_dataset_json(cls): rv = cls.app.get('/datasets/%s/json' % cls.dataset_id) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - return json.loads(rv.data) + return json.loads(rv.get_data(as_text=True)) @classmethod def get_entry_count(cls, stage): @@ -138,7 +138,7 @@ def create_random_imageset(cls, **kwargs): if not hasattr(cls, 'imageset_folder'): # create a temporary folder cls.imageset_folder = tempfile.mkdtemp() - for i in xrange(num_images): + for i in range(num_images): x = np.random.randint( low=0, high=256, @@ -162,7 +162,7 @@ def create_variable_size_random_imageset(cls, **kwargs): if not hasattr(cls, 'imageset_folder'): # create a temporary folder cls.imageset_folder = tempfile.mkdtemp() - for i in xrange(num_images): + for i in range(num_images): image_width = np.random.randint(low=8, high=32) image_height = np.random.randint(low=8, high=32) x = np.random.randint( @@ -207,9 +207,9 @@ def setUpClass(cls, **kwargs): def test_page_dataset_new(self): rv = self.app.get('/datasets/generic/new/%s' % self.EXTENSION_ID) - print rv.data + print(rv.get_data(as_text=True)) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert extensions.data.get_extension(self.EXTENSION_ID).get_title() in rv.data, 'unexpected page format' + assert extensions.data.get_extension(self.EXTENSION_ID).get_title() in rv.get_data(as_text=True), 'unexpected page format' def test_nonexistent_dataset(self): assert not self.dataset_exists('foo'), "dataset shouldn't exist" @@ -264,7 +264,7 @@ def test_clone(self): assert self.dataset_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/datasets/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content1 = json.loads(rv.data) + content1 = json.loads(rv.get_data(as_text=True)) # Clone job1 as job2 options_2 = { @@ -275,7 +275,7 @@ def test_clone(self): assert self.dataset_wait_completion(job2_id) == 'Done', 'second job failed' rv = self.app.get('/datasets/%s/json' % job2_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content2 = json.loads(rv.data) + content2 = json.loads(rv.get_data(as_text=True)) # These will be different content1.pop('id') @@ -298,7 +298,7 @@ class GenericCreatedTest(BaseViewsTestWithDataset): def test_index_json(self): rv = self.app.get('/index/json') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) found = False for d in content['datasets']: if d['id'] == self.dataset_id: @@ -318,7 +318,7 @@ def test_edit_name(self): assert status == 200, 'failed with %s' % status rv = self.app.get('/datasets/summary?job_id=%s' % self.dataset_id) assert rv.status_code == 200 - assert 'new name' in rv.data + assert 'new name' in rv.get_data(as_text=True) def test_edit_notes(self): status = self.edit_job( diff --git a/digits/dataset/generic/views.py b/digits/dataset/generic/views.py index a63477f2b..5c8df2424 100644 --- a/digits/dataset/generic/views.py +++ b/digits/dataset/generic/views.py @@ -6,18 +6,20 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO - -import caffe_pb2 + from io import StringIO +from io import BytesIO +# import caffe_pb2 import flask import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import PIL.Image +import base64 from .forms import GenericDatasetForm from .job import GenericDatasetJob from digits import extensions, utils +from digits.dataset import dataset_pb2 from digits.utils.constants import COLOR_PALETTE_ATTRIBUTE from digits.utils.routing import request_wants_json, job_from_request from digits.utils.lmdbreader import DbReader @@ -172,16 +174,16 @@ def explore(): min_page = max(0, page - 5) total_entries = reader.total_entries - max_page = min((total_entries - 1) / size, page + 5) + max_page = min((total_entries - 1) // size, page + 5) pages = range(min_page, max_page + 1) for key, value in reader.entries(): if count >= page * size: - datum = caffe_pb2.Datum() + datum = dataset_pb2.Datum() datum.ParseFromString(value) if not datum.encoded: raise RuntimeError("Expected encoded database") s = StringIO() - s.write(datum.data) + s.write(datum.data.decode()) s.seek(0) img = PIL.Image.open(s) if cmap and img.mode in ['L', '1']: From e4f7d1f596413f9acb30595242331624a1ac9103 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 12:47:46 -0400 Subject: [PATCH 03/21] Update for Python3 --- digits/dataset/images/classification/job.py | 6 +-- .../classification/test_imageset_creator.py | 4 +- .../images/classification/test_views.py | 54 ++++++++++--------- digits/dataset/images/classification/views.py | 23 +++++--- .../images/generic/test_lmdb_creator.py | 24 ++++----- digits/dataset/images/generic/test_views.py | 20 +++---- digits/dataset/images/views.py | 2 +- 7 files changed, 71 insertions(+), 62 deletions(-) diff --git a/digits/dataset/images/classification/job.py b/digits/dataset/images/classification/job.py index eb61e7e41..7ca785d30 100644 --- a/digits/dataset/images/classification/job.py +++ b/digits/dataset/images/classification/job.py @@ -33,9 +33,9 @@ def __setstate__(self, state): if task.encoding == "jpg": if task.mean_file.endswith('.binaryproto'): import numpy as np - import caffe_pb2 + from digits.dataset import dataset_pb2 - old_blob = caffe_pb2.BlobProto() + old_blob = dataset_pb2.BlobProto() with open(task.path(task.mean_file), 'rb') as infile: old_blob.ParseFromString(infile.read()) data = np.array(old_blob.data).reshape( @@ -43,7 +43,7 @@ def __setstate__(self, state): old_blob.height, old_blob.width) data = data[[2, 1, 0], ...] # channel swap - new_blob = caffe_pb2.BlobProto() + new_blob = dataset_pb2.BlobProto() new_blob.num = 1 new_blob.channels, new_blob.height, new_blob.width = data.shape new_blob.data.extend(data.astype(float).flat) diff --git a/digits/dataset/images/classification/test_imageset_creator.py b/digits/dataset/images/classification/test_imageset_creator.py index 1c5fcbd2b..5752c985e 100755 --- a/digits/dataset/images/classification/test_imageset_creator.py +++ b/digits/dataset/images/classification/test_imageset_creator.py @@ -93,7 +93,7 @@ def _create_gradient_image(size, color_from, color_to, rotation): args = vars(parser.parse_args()) - print 'Creating images at "%s" ...' % args['folder'] + print('Creating images at "%s" ...' % args['folder']) start_time = time.time() @@ -102,4 +102,4 @@ def _create_gradient_image(size, color_from, color_to, rotation): image_count=args['image_count'], ) - print 'Done after %s seconds' % (time.time() - start_time,) + print('Done after %s seconds' % (time.time() - start_time,)) diff --git a/digits/dataset/images/classification/test_views.py b/digits/dataset/images/classification/test_views.py index 2fc26c776..171ec3dcb 100644 --- a/digits/dataset/images/classification/test_views.py +++ b/digits/dataset/images/classification/test_views.py @@ -10,7 +10,8 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO +from io import BytesIO from bs4 import BeautifulSoup import PIL.Image @@ -19,6 +20,7 @@ from digits import test_utils import digits.test_views +import pdb # May be too short on a slow system TIMEOUT_DATASET = 45 @@ -128,18 +130,18 @@ def create_dataset(cls, **kwargs): if request_json: if rv.status_code != 200: - print json.loads(rv.data) + print(json.loads(rv.get_data(as_text=True))) raise RuntimeError('Model creation failed with %s' % rv.status_code) - return json.loads(rv.data)['id'] + return json.loads(rv.get_data(as_text=True))['id'] # expect a redirect if not 300 <= rv.status_code <= 310: - s = BeautifulSoup(rv.data, 'html.parser') + s = BeautifulSoup(rv.get_data(as_text=True), 'html.parser') div = s.select('div.alert-danger') if div: - print div[0] + print(div[0]) else: - print rv.data + print(rv.get_data(as_text=True)) raise RuntimeError('Failed to create dataset - status %s' % rv.status_code) job_id = cls.job_id_from_response(rv) @@ -183,7 +185,7 @@ def test_clone(self): assert self.dataset_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/datasets/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content1 = json.loads(rv.data) + content1 = json.loads(rv.get_data(as_text=True)) # Clone job1 as job2 options_2 = { @@ -194,7 +196,7 @@ def test_clone(self): assert self.dataset_wait_completion(job2_id) == 'Done', 'second job failed' rv = self.app.get('/datasets/%s/json' % job2_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content2 = json.loads(rv.data) + content2 = json.loads(rv.get_data(as_text=True)) # These will be different content1.pop('id') @@ -220,7 +222,7 @@ class TestViews(BaseViewsTest, test_utils.DatasetMixin): def test_page_dataset_new(self): rv = self.app.get('/datasets/images/classification/new') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'New Image Classification Dataset' in rv.data, 'unexpected page format' + assert 'New Image Classification Dataset' in rv.get_data(as_text=True), 'unexpected page format' def test_nonexistent_dataset(self): assert not self.dataset_exists('foo'), "dataset shouldn't exist" @@ -276,7 +278,7 @@ def check_textfiles(self, absolute_path=True, local_path=True): textfile_train_images = '' textfile_labels_file = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): textfile_labels_file += '%s\n' % label for image in images: image_path = image @@ -306,10 +308,10 @@ def check_textfiles(self, absolute_path=True, local_path=True): data['textfile_local_labels_file'] = labels_file else: # StringIO wrapping is needed to simulate POST file upload. - train_upload = (StringIO(textfile_train_images), "train.txt") + train_upload = (BytesIO(textfile_train_images.encode('utf-8')), "train.txt") # Use the same list for training and validation. - val_upload = (StringIO(textfile_train_images), "val.txt") - labels_upload = (StringIO(textfile_labels_file), "labels.txt") + val_upload = (BytesIO(textfile_train_images.encode('utf-8')), "val.txt") + labels_upload = (BytesIO(textfile_labels_file.encode('utf-8')), "labels.txt") data['textfile_train_images'] = train_upload data['textfile_val_images'] = val_upload data['textfile_labels_file'] = labels_upload @@ -326,7 +328,7 @@ def test_abort_explore_fail(self): self.abort_dataset(job_id) rv = self.app.get('/datasets/images/classification/explore?job_id=%s&db=val' % job_id) assert rv.status_code == 500, 'page load should have failed' - assert 'status should be' in rv.data, 'unexpected page format' + assert 'status should be' in rv.get_data(as_text=True), 'unexpected page format' class TestImageCount(BaseViewsTestWithImageset, test_utils.DatasetMixin): @@ -381,7 +383,7 @@ def test_max_per_class(self): def check_max_per_class(self, type): # create dataset, asking for at most IMAGE_COUNT/2 images per class assert self.IMAGE_COUNT % 2 == 0 - max_per_class = self.IMAGE_COUNT / 2 + max_per_class = self.IMAGE_COUNT // 2 data = {'folder_pct_val': 0} if type == 'train': data['folder_train_max_per_class'] = max_per_class @@ -422,7 +424,7 @@ def test_min_per_class(self): def check_min_per_class(self, type): # create dataset, asking for one more image per class # than available in the "unbalanced" category - min_per_class = self.IMAGE_COUNT / 2 + 1 + min_per_class = self.IMAGE_COUNT // 2 + 1 data = {'folder_pct_val': 0} if type == 'train': data['folder_train_min_per_class'] = min_per_class @@ -459,7 +461,7 @@ class TestCreated(BaseViewsTestWithDataset, test_utils.DatasetMixin): def test_index_json(self): rv = self.app.get('/index/json') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) found = False for d in content['datasets']: if d['id'] == self.dataset_id: @@ -470,14 +472,14 @@ def test_index_json(self): def test_dataset_json(self): rv = self.app.get('/datasets/%s/json' % self.dataset_id) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert content['id'] == self.dataset_id, 'expected different job_id' def test_mean_dimensions(self): img_url = '/files/%s/mean.jpg' % self.dataset_id rv = self.app.get(img_url) assert rv.status_code == 200, 'GET on %s returned %s' % (img_url, rv.status_code) - buff = StringIO(rv.data) + buff = BytesIO(rv.data) buff.seek(0) pil_image = PIL.Image.open(buff) assert pil_image.size == (self.IMAGE_WIDTH, self.IMAGE_HEIGHT), 'image size is %s' % (pil_image.size,) @@ -489,7 +491,7 @@ def test_edit_name(self): assert status == 200, 'failed with %s' % status rv = self.app.get('/datasets/summary?job_id=%s' % self.dataset_id) assert rv.status_code == 200 - assert 'new name' in rv.data + assert 'new name' in rv.get_data(as_text=True) def test_edit_notes(self): status = self.edit_job( @@ -500,7 +502,7 @@ def test_edit_notes(self): def test_backend_selection(self): rv = self.app.get('/datasets/%s/json' % self.dataset_id) - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) for task in content['CreateDbTasks']: assert task['backend'] == self.BACKEND @@ -509,20 +511,20 @@ def test_explore_train(self): if self.BACKEND == 'hdf5': # Not supported yet assert rv.status_code == 500, 'page load should have failed' - assert 'expected backend is lmdb' in rv.data, 'unexpected page format' + assert 'expected backend is lmdb' in rv.get_data(as_text=True), 'unexpected page format' else: assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'Items per page' in rv.data, 'unexpected page format' + assert 'Items per page' in rv.get_data(as_text=True), 'unexpected page format' def test_explore_val(self): rv = self.app.get('/datasets/images/classification/explore?job_id=%s&db=val' % self.dataset_id) if self.BACKEND == 'hdf5': # Not supported yet assert rv.status_code == 500, 'page load should have failed' - assert 'expected backend is lmdb' in rv.data, 'unexpected page format' + assert 'expected backend is lmdb' in rv.get_data(as_text=True), 'unexpected page format' else: assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'Items per page' in rv.data, 'unexpected page format' + assert 'Items per page' in rv.get_data(as_text=True), 'unexpected page format' class TestCreatedGrayscale(TestCreated, test_utils.DatasetMixin): @@ -555,7 +557,7 @@ class TestCreatedHdf5(TestCreated, test_utils.DatasetMixin): def test_compression_method(self): rv = self.app.get('/datasets/%s/json' % self.dataset_id) - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) for task in content['CreateDbTasks']: assert task['compression'] == self.COMPRESSION diff --git a/digits/dataset/images/classification/views.py b/digits/dataset/images/classification/views.py index 23316fc03..998d24801 100644 --- a/digits/dataset/images/classification/views.py +++ b/digits/dataset/images/classification/views.py @@ -8,9 +8,9 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO +from io import BytesIO -import caffe_pb2 import flask import PIL.Image @@ -18,6 +18,8 @@ from .job import ImageClassificationDatasetJob from digits import utils from digits.dataset import tasks +from digits.dataset import dataset_pb2 +from digits.dataset.datum import datum_to_array from digits.utils.forms import fill_form_if_cloned, save_form_to_job from digits.utils.lmdbreader import DbReader from digits.utils.routing import request_wants_json, job_from_request @@ -48,6 +50,8 @@ def from_folders(job, form): min_per_class = form.folder_train_min_per_class.data max_per_class = form.folder_train_max_per_class.data + if max_per_class is None: + max_per_class = 0 parse_train_task = tasks.ParseFolderTask( job_dir=job.dir(), folder=form.folder_train.data, @@ -68,6 +72,8 @@ def from_folders(job, form): min_per_class = form.folder_val_min_per_class.data max_per_class = form.folder_val_max_per_class.data + if max_per_class is None: + max_per_class = 0 parse_val_task = tasks.ParseFolderTask( job_dir=job.dir(), parents=parse_train_task, @@ -84,6 +90,8 @@ def from_folders(job, form): min_per_class = form.folder_test_min_per_class.data max_per_class = form.folder_test_max_per_class.data + if max_per_class is None: + max_per_class = 0 parse_test_task = tasks.ParseFolderTask( job_dir=job.dir(), parents=parse_train_task, @@ -498,21 +506,20 @@ def explore(): else: total_entries = label_entries - max_page = min((total_entries - 1) / size, page + 5) + max_page = min(total_entries - 1 // size, page + 5) pages = range(min_page, max_page + 1) for key, value in reader.entries(): if count >= page * size: - datum = caffe_pb2.Datum() + datum = dataset_pb2.Datum() datum.ParseFromString(value) if label is None or datum.label == label: if datum.encoded: - s = StringIO() + s = BytesIO() s.write(datum.data) s.seek(0) img = PIL.Image.open(s) else: - import caffe.io - arr = caffe.io.datum_to_array(datum) + arr = datum_to_array(datum) # CHW -> HWC arr = arr.transpose((1, 2, 0)) if arr.shape[2] == 1: @@ -527,7 +534,7 @@ def explore(): if label is None: count += 1 else: - datum = caffe_pb2.Datum() + datum = dataset_pb2.Datum() datum.ParseFromString(value) if datum.label == int(label): count += 1 diff --git a/digits/dataset/images/generic/test_lmdb_creator.py b/digits/dataset/images/generic/test_lmdb_creator.py index 09f638524..d6dcbd754 100755 --- a/digits/dataset/images/generic/test_lmdb_creator.py +++ b/digits/dataset/images/generic/test_lmdb_creator.py @@ -15,7 +15,7 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO import lmdb import numpy as np @@ -27,7 +27,7 @@ import digits.config # noqa # Import digits.config first to set the path to Caffe -import caffe_pb2 # noqa +from digits.dataset import dataset_pb2 # noqa IMAGE_SIZE = 10 @@ -74,7 +74,7 @@ def create_lmdbs(folder, image_width=None, image_height=None, image_count=None): image_sum = np.zeros((image_height, image_width), 'float64') - for i in xrange(image_count): + for i in range(image_count): xslope, yslope = np.random.random_sample(2) - 0.5 a = xslope * 255 / image_width b = yslope * 255 / image_height @@ -86,18 +86,18 @@ def create_lmdbs(folder, image_width=None, image_height=None, image_count=None): pil_img = PIL.Image.fromarray(image) # create image Datum - image_datum = caffe_pb2.Datum() + image_datum = dataset_pb2.Datum() image_datum.height = image.shape[0] image_datum.width = image.shape[1] image_datum.channels = 1 s = StringIO() - pil_img.save(s, format='PNG') - image_datum.data = s.getvalue() + pil_img.save(str(s), format='PNG') + image_datum.data = s.getvalue().encode('utf-8') image_datum.encoded = True _write_to_lmdb(image_db, str(i), image_datum.SerializeToString()) # create label Datum - label_datum = caffe_pb2.Datum() + label_datum = dataset_pb2.Datum() label_datum.channels, label_datum.height, label_datum.width = 1, 1, 2 label_datum.float_data.extend(np.array([xslope, yslope]).flat) _write_to_lmdb(label_db, str(i), label_datum.SerializeToString()) @@ -133,7 +133,7 @@ def _write_to_lmdb(db, key, value): while not success: txn = db.begin(write=True) try: - txn.put(key, value) + txn.put(key.encode('utf-8'), value) txn.commit() success = True except lmdb.MapFullError: @@ -154,7 +154,7 @@ def _save_mean(mean, filename): filename -- the location to save the image """ if filename.endswith('.binaryproto'): - blob = caffe_pb2.BlobProto() + blob = dataset_pb2.BlobProto() blob.num = 1 blob.channels = 1 blob.height, blob.width = mean.shape @@ -193,12 +193,12 @@ def _save_mean(mean, filename): args = vars(parser.parse_args()) if os.path.exists(args['folder']): - print 'ERROR: Folder already exists' + print('ERROR: Folder already exists') sys.exit(1) else: os.makedirs(args['folder']) - print 'Creating images at "%s" ...' % args['folder'] + print('Creating images at "%s" ...' % args['folder']) start_time = time.time() @@ -208,4 +208,4 @@ def _save_mean(mean, filename): image_count=args['image_count'], ) - print 'Done after %s seconds' % (time.time() - start_time,) + print('Done after %s seconds' % (time.time() - start_time,)) diff --git a/digits/dataset/images/generic/test_views.py b/digits/dataset/images/generic/test_views.py index 6eac8fe6b..654147b57 100644 --- a/digits/dataset/images/generic/test_views.py +++ b/digits/dataset/images/generic/test_views.py @@ -102,18 +102,18 @@ def create_dataset(cls, **kwargs): if request_json: if rv.status_code != 200: - print json.loads(rv.data) + print(json.loads(rv.get_data(as_text=True))) raise RuntimeError('Model creation failed with %s' % rv.status_code) - return json.loads(rv.data)['id'] + return json.loads(rv.get_data(as_text=True))['id'] # expect a redirect if not 300 <= rv.status_code <= 310: s = BeautifulSoup(rv.data, 'html.parser') div = s.select('div.alert-danger') if div: - print div[0] + print(div[0]) else: - print rv.data + print(rv.get_data(as_text=True)) raise RuntimeError('Failed to create dataset - status %s' % rv.status_code) job_id = cls.job_id_from_response(rv) @@ -147,7 +147,7 @@ class TestViews(BaseViewsTest, test_utils.DatasetMixin): def test_page_dataset_new(self): rv = self.app.get('/datasets/images/generic/new') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'New Image Dataset' in rv.data, 'unexpected page format' + assert 'New Image Dataset' in rv.get_data(as_text=True), 'unexpected page format' def test_nonexistent_dataset(self): assert not self.dataset_exists('foo'), "dataset shouldn't exist" @@ -201,7 +201,7 @@ def test_clone(self): assert self.dataset_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/datasets/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content1 = json.loads(rv.data) + content1 = json.loads(rv.get_data(as_text=True)) # Clone job1 as job2 options_2 = { @@ -212,7 +212,7 @@ def test_clone(self): assert self.dataset_wait_completion(job2_id) == 'Done', 'second job failed' rv = self.app.get('/datasets/%s/json' % job2_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content2 = json.loads(rv.data) + content2 = json.loads(rv.get_data(as_text=True)) # These will be different content1.pop('id') @@ -235,7 +235,7 @@ class TestCreated(BaseViewsTestWithDataset, test_utils.DatasetMixin): def test_index_json(self): rv = self.app.get('/index/json') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) found = False for d in content['datasets']: if d['id'] == self.dataset_id: @@ -246,7 +246,7 @@ def test_index_json(self): def test_dataset_json(self): rv = self.app.get('/datasets/%s/json' % self.dataset_id) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert content['id'] == self.dataset_id, 'expected different job_id' def test_edit_name(self): @@ -257,7 +257,7 @@ def test_edit_name(self): assert status == 200, 'failed with %s' % status rv = self.app.get('/datasets/summary?job_id=%s' % self.dataset_id) assert rv.status_code == 200 - assert 'new name' in rv.data + assert 'new name' in rv.get_data(as_text=True) def test_edit_notes(self): status = self.edit_job( diff --git a/digits/dataset/images/views.py b/digits/dataset/images/views.py index fdef36c36..a502d4b6f 100644 --- a/digits/dataset/images/views.py +++ b/digits/dataset/images/views.py @@ -7,7 +7,7 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO import flask import PIL.Image From 14b6ded08abdfa053b47f3db7cc30183e6db7f9b Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 12:49:17 -0400 Subject: [PATCH 04/21] Update for Python3 From 76f34ade43fc9af39a55b289285f087af7ca6e76 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 12:50:32 -0400 Subject: [PATCH 05/21] Update for Python3 --- digits/download_data/__main__.py | 4 ++-- digits/download_data/cifar10.py | 10 +++++----- digits/download_data/cifar100.py | 10 +++++----- digits/download_data/mnist.py | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/digits/download_data/__main__.py b/digits/download_data/__main__.py index 83498e249..bdffbae4e 100644 --- a/digits/download_data/__main__.py +++ b/digits/download_data/__main__.py @@ -48,7 +48,7 @@ clean=args['clean']) d.getData() else: - print 'Unknown dataset "%s"' % args['dataset'] + print('Unknown dataset "%s"' % args['dataset']) sys.exit(1) - print 'Done after %s seconds.' % (time.time() - start) + print('Done after %s seconds.' % (time.time() - start)) diff --git a/digits/download_data/cifar10.py b/digits/download_data/cifar10.py index 0d50b3787..027518c39 100644 --- a/digits/download_data/cifar10.py +++ b/digits/download_data/cifar10.py @@ -1,6 +1,6 @@ # Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. -import cPickle +import pickle import os import tarfile @@ -26,7 +26,7 @@ def uncompressData(self): assert os.path.exists(filepath), 'Expected "%s" to exist' % filename if not os.path.exists(os.path.join(self.outdir, 'cifar-10-batches-py')): - print "Uncompressing file=%s ..." % filename + print("Uncompressing file=%s ..." % filename) with tarfile.open(filepath) as tf: tf.extractall(self.outdir) @@ -34,7 +34,7 @@ def processData(self): label_filename = 'batches.meta' label_filepath = os.path.join(self.outdir, 'cifar-10-batches-py', label_filename) with open(label_filepath, 'rb') as infile: - pickleObj = cPickle.load(infile) + pickleObj = pickle.load(infile) label_names = pickleObj['label_names'] for phase in 'train', 'test': @@ -66,11 +66,11 @@ def __extractData(self, input_file, phase, label_names): phase -- train or test label_names -- a list of strings """ - print 'Extracting images file=%s ...' % input_file + print('Extracting images file=%s ...' % input_file) # Read the pickle file with open(input_file, 'rb') as infile: - pickleObj = cPickle.load(infile) + pickleObj = pickle.load(infile) # print 'Batch -', pickleObj['batch_label'] data = pickleObj['data'] assert data.shape == (10000, 3072), 'Expected data.shape to be (10000, 3072), not %s' % (data.shape,) diff --git a/digits/download_data/cifar100.py b/digits/download_data/cifar100.py index 1ede1ce76..48fd7def0 100644 --- a/digits/download_data/cifar100.py +++ b/digits/download_data/cifar100.py @@ -1,6 +1,6 @@ # Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. -import cPickle +import pickle import os import tarfile @@ -26,7 +26,7 @@ def uncompressData(self): assert os.path.exists(filepath), 'Expected "%s" to exist' % filename if not os.path.exists(os.path.join(self.outdir, 'cifar-100-python')): - print "Uncompressing file=%s ..." % filename + print("Uncompressing file=%s ..." % filename) with tarfile.open(filepath) as tf: tf.extractall(self.outdir) @@ -34,7 +34,7 @@ def processData(self): label_filename = 'meta' label_filepath = os.path.join(self.outdir, 'cifar-100-python', label_filename) with open(label_filepath, 'rb') as infile: - pickleObj = cPickle.load(infile) + pickleObj = pickle.load(infile) fine_label_names = pickleObj['fine_label_names'] coarse_label_names = pickleObj['coarse_label_names'] @@ -67,11 +67,11 @@ def __extractData(self, input_file, phase, fine_label_names, coarse_label_names) fine_label_names -- mapping from fine_labels to strings coarse_label_names -- mapping from coarse_labels to strings """ - print 'Extracting images file=%s ...' % input_file + print('Extracting images file=%s ...' % input_file) # Read the pickle file with open(input_file, 'rb') as infile: - pickleObj = cPickle.load(infile) + pickleObj = pickle.load(infile) # print 'Batch -', pickleObj['batch_label'] data = pickleObj['data'] assert data.shape[1] == 3072, 'Unexpected data.shape %s' % (data.shape,) diff --git a/digits/download_data/mnist.py b/digits/download_data/mnist.py index e858b5ddb..6d7369039 100644 --- a/digits/download_data/mnist.py +++ b/digits/download_data/mnist.py @@ -35,7 +35,7 @@ def uncompressData(self): assert os.path.exists(zipped_path), 'Expected "%s" to exist' % zipped unzipped_path = os.path.join(self.outdir, unzipped) if not os.path.exists(unzipped_path): - print "Uncompressing file=%s ..." % zipped + print("Uncompressing file=%s ..." % zipped) with gzip.open(zipped_path) as infile, open(unzipped_path, 'wb') as outfile: outfile.write(infile.read()) @@ -54,7 +54,7 @@ def __extract_images(self, images_file, labels_file, phase): output_dir = os.path.join(self.outdir, phase) self.mkdir(output_dir, clean=True) with open(os.path.join(output_dir, 'labels.txt'), 'w') as outfile: - for label in xrange(10): + for label in range(10): outfile.write('%s\n' % label) with open(os.path.join(output_dir, '%s.txt' % phase), 'w') as outfile: for index, image in enumerate(images): @@ -68,7 +68,7 @@ def __readLabels(self, filename): """ Returns a list of ints """ - print 'Reading labels from %s ...' % filename + print('Reading labels from %s ...' % filename) labels = [] with open(filename, 'rb') as infile: infile.read(4) # ignore magic number @@ -83,7 +83,7 @@ def __readImages(self, filename): """ Returns a list of PIL.Image objects """ - print 'Reading images from %s ...' % filename + print('Reading images from %s ...' % filename) images = [] with open(filename, 'rb') as infile: infile.read(4) # ignore magic number @@ -91,7 +91,7 @@ def __readImages(self, filename): rows = struct.unpack('>i', infile.read(4))[0] columns = struct.unpack('>i', infile.read(4))[0] - for i in xrange(count): + for i in range(count): data = infile.read(rows * columns) image = np.fromstring(data, dtype=np.uint8) image = image.reshape((rows, columns)) From a503b18e875cb2f59823ff231b100f5ba4d694b9 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:49:01 -0400 Subject: [PATCH 06/21] Update for Python3 --- digits/extensions/data/imageProcessing/data.py | 4 ++-- digits/extensions/data/objectDetection/data.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/digits/extensions/data/imageProcessing/data.py b/digits/extensions/data/imageProcessing/data.py index cb03a19be..8efa7e713 100644 --- a/digits/extensions/data/imageProcessing/data.py +++ b/digits/extensions/data/imageProcessing/data.py @@ -108,7 +108,7 @@ def get_title(): @override def itemize_entries(self, stage): if stage == constants.TEST_DB: - # don't retun anything for the test stage + # don't return anything for the test stage return [] if stage == constants.TRAIN_DB or (not self.has_val_folder): @@ -156,7 +156,7 @@ def make_image_list(self, folder): def split_image_list(self, filelist, stage): if self.random_indices is None: self.random_indices = range(len(filelist)) - random.shuffle(self.random_indices) + random.shuffle(list(self.random_indices)) elif len(filelist) != len(self.random_indices): raise ValueError( "Expect same number of images in folders (%d!=%d)" diff --git a/digits/extensions/data/objectDetection/data.py b/digits/extensions/data/objectDetection/data.py index 31cdc1859..46c2b8cbe 100644 --- a/digits/extensions/data/objectDetection/data.py +++ b/digits/extensions/data/objectDetection/data.py @@ -5,7 +5,7 @@ import operator import os import random -import StringIO +import io import numpy as np @@ -32,7 +32,7 @@ def __init__(self, **kwargs): # attributes by superclass constructor if hasattr(self, 'custom_classes') and self.custom_classes != '': - s = StringIO.StringIO(self.custom_classes) + s = io.StringIO(self.custom_classes) reader = csv.reader(s) self.class_mappings = {} for idx, name in enumerate(reader.next()): From ff052d04c0f780b2da19d9dddaba90b8e92724db Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:52:09 -0400 Subject: [PATCH 07/21] Update for Python3 --- digits/extensions/view/imageSegmentation/header_template.html | 4 ++-- digits/extensions/view/imageSegmentation/view.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/digits/extensions/view/imageSegmentation/header_template.html b/digits/extensions/view/imageSegmentation/header_template.html index 30295f581..1385cc67f 100644 --- a/digits/extensions/view/imageSegmentation/header_template.html +++ b/digits/extensions/view/imageSegmentation/header_template.html @@ -1,4 +1,4 @@ {# Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. #} - - + + diff --git a/digits/extensions/view/imageSegmentation/view.py b/digits/extensions/view/imageSegmentation/view.py index eb425d57f..491fff9c3 100644 --- a/digits/extensions/view/imageSegmentation/view.py +++ b/digits/extensions/view/imageSegmentation/view.py @@ -194,7 +194,7 @@ def process_data(self, input_id, input_data, output_data): fill_data = (self.map.to_rgba(class_data) * 255).astype('uint8') else: fill_data = np.ndarray((class_data.shape[0], class_data.shape[1], 4), dtype='uint8') - for x in xrange(3): + for x in range(3): fill_data[:, :, x] = class_data.copy() # Assuming that class 0 is the background From fa69f11d23ac7ff4255f4cda4983628ab3083a82 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:53:37 -0400 Subject: [PATCH 08/21] Update for Python3 --- digits/frameworks/__init__.py | 37 +++++++++++++++------------------- digits/frameworks/framework.py | 16 +++++++++++++++ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/digits/frameworks/__init__.py b/digits/frameworks/__init__.py index a6eb057f4..78332e4f2 100644 --- a/digits/frameworks/__init__.py +++ b/digits/frameworks/__init__.py @@ -1,34 +1,27 @@ # Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. from __future__ import absolute_import -from .caffe_framework import CaffeFramework from .framework import Framework -from .torch_framework import TorchFramework from digits.config import config_value __all__ = [ - 'Framework', - 'CaffeFramework', - 'TorchFramework', ] +caffe = None +tensorflow = None + +if config_value('caffe')['loaded']: + from .caffe_framework import CaffeFramework + caffe = CaffeFramework() + __all__.append('CaffeFramework') + if config_value('tensorflow')['enabled']: from .tensorflow_framework import TensorflowFramework + tensorflow = TensorflowFramework() __all__.append('TensorflowFramework') -# -# create framework instances -# - -# torch is optional -torch = TorchFramework() if config_value('torch')['enabled'] else None - -# tensorflow is optional -tensorflow = TensorflowFramework() if config_value('tensorflow')['enabled'] else None - -# caffe is mandatory -caffe = CaffeFramework() - +if len(__all__) == 0: + __all__.append('Framework') # # utility functions # @@ -39,11 +32,13 @@ def get_frameworks(): return list of all available framework instances there may be more than one instance per framework class """ - frameworks = [caffe] - if torch: - frameworks.append(torch) + frameworks = [] + if caffe: + frameworks.append(caffe) if tensorflow: frameworks.append(tensorflow) + if len(frameworks) == 0: + frameworks.append(Framework()) return frameworks diff --git a/digits/frameworks/framework.py b/digits/frameworks/framework.py index ba6d469ee..1f7d3df78 100644 --- a/digits/frameworks/framework.py +++ b/digits/frameworks/framework.py @@ -8,6 +8,22 @@ class Framework(object): Defines required methods to interact with a framework """ + # whether this framework can shuffle data during training + CAN_SHUFFLE_DATA = False + SUPPORTS_PYTHON_LAYERS_FILE = False + SUPPORTS_TIMELINE_TRACING = False + + SUPPORTED_SOLVER_TYPES = [] + + SUPPORTED_DATA_TRANSFORMATION_TYPES = [] + SUPPORTED_DATA_AUGMENTATION_TYPES = [] + + NAME = 'Name_FRAMEWORK' + + def __init__(self): + self.framework_id = 'Id_FRAMEWORK' + + def get_name(self): """ return self-descriptive name From 086b818c5036624ab3bece1602238194e18d4aeb Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:55:26 -0400 Subject: [PATCH 09/21] Update for Python3 --- digits/inference/tasks/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digits/inference/tasks/inference.py b/digits/inference/tasks/inference.py index 0b8d0e097..8165862e0 100644 --- a/digits/inference/tasks/inference.py +++ b/digits/inference/tasks/inference.py @@ -75,7 +75,7 @@ def before_run(self): # create a file to pass the list of images to perform inference on imglist_handle, self.image_list_path = tempfile.mkstemp(dir=self.job_dir, suffix='.txt') for image_path in self.images: - os.write(imglist_handle, "%s\n" % image_path) + os.write(imglist_handle, ("%s\n" % image_path).encode('utf-8')) os.close(imglist_handle) @override From 77b0d651176cc26267a4412986419599bc127631 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:57:19 -0400 Subject: [PATCH 10/21] Update for Python3 --- .../model/images/classification/test_views.py | 99 ++++++++++--------- digits/model/images/classification/views.py | 22 +++-- digits/model/images/generic/test_views.py | 49 ++++----- digits/model/images/generic/views.py | 16 +-- 4 files changed, 99 insertions(+), 87 deletions(-) diff --git a/digits/model/images/classification/test_views.py b/digits/model/images/classification/test_views.py index d63fbdc00..553318a1b 100644 --- a/digits/model/images/classification/test_views.py +++ b/digits/model/images/classification/test_views.py @@ -8,7 +8,6 @@ import tempfile import time import unittest -import caffe_pb2 import math @@ -16,8 +15,9 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO +from io import BytesIO from bs4 import BeautifulSoup from digits.config import config_value @@ -25,9 +25,12 @@ import digits.test_views from digits import test_utils import digits.webapp -from digits.frameworks import CaffeFramework + from google.protobuf import text_format +if config_value('caffe')['loaded']: + from digits.frameworks import CaffeFramework + from caffe.proto import caffe_pb2 # May be too short on a slow system TIMEOUT_DATASET = 45 TIMEOUT_MODEL = 60 @@ -180,6 +183,7 @@ class BaseViewsTestWithDataset(BaseViewsTest, AUG_HSV_S = None AUG_HSV_V = None OPTIMIZER = None + BLOB_FORMAT = None @classmethod def setUpClass(cls): @@ -215,7 +219,8 @@ def create_model(cls, network=None, **kwargs): 'train_epochs': cls.TRAIN_EPOCHS, 'framework': cls.FRAMEWORK, 'random_seed': 0xCAFEBABE, - 'shuffle': 'true' if cls.SHUFFLE else 'false' + 'shuffle': 'true' if cls.SHUFFLE else 'false', + 'blob_format': 'NVCaffe' } if cls.CROP_SIZE is not None: data['crop_size'] = cls.CROP_SIZE @@ -262,9 +267,9 @@ def create_model(cls, network=None, **kwargs): if request_json: if rv.status_code != 200: - print json.loads(rv.data) + print(json.loads(rv.get_data(as_text=True))) raise RuntimeError('Model creation failed with %s' % rv.status_code) - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) if 'jobs' in data.keys(): return [j['id'] for j in data['jobs']] else: @@ -272,13 +277,13 @@ def create_model(cls, network=None, **kwargs): # expect a redirect if not 300 <= rv.status_code <= 310: - print 'Status code:', rv.status_code + print('Status code:', rv.status_code) s = BeautifulSoup(rv.data, 'html.parser') div = s.select('div.alert-danger') if div: - print div[0] + print(div[0]) else: - print rv.data + print(rv.get_data(as_text=True)) raise RuntimeError('Failed to create dataset - status %s' % rv.status_code) job_id = cls.job_id_from_response(rv) @@ -307,7 +312,7 @@ class BaseTestViews(BaseViewsTest): def test_page_model_new(self): rv = self.app.get('/models/images/classification/new') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'New Image Classification Model' in rv.data, 'unexpected page format' + assert 'New Image Classification Model' in rv.get_data(as_text=True), 'unexpected page format' def test_nonexistent_model(self): assert not self.model_exists('foo'), "model shouldn't exist" @@ -363,7 +368,7 @@ def test_snapshot_interval_2(self): assert self.model_wait_completion(job_id) == 'Done', 'create failed' rv = self.app.get('/models/%s/json' % job_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']) > 1, 'should take >1 snapshot' def test_snapshot_interval_0_5(self): @@ -371,7 +376,7 @@ def test_snapshot_interval_0_5(self): assert self.model_wait_completion(job_id) == 'Done', 'create failed' rv = self.app.get('/models/%s/json' % job_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']) == 2, 'should take 2 snapshots' @unittest.skipIf( @@ -403,9 +408,11 @@ def check_select_gpu(self, gpu_index): def test_select_gpus(self): # test all possible combinations gpu_list = config_value('gpu_list').split(',') - for i in xrange(len(gpu_list)): + for i in range(len(gpu_list)): for combination in itertools.combinations(gpu_list, i + 1): - yield self.check_select_gpus, combination + # Don't test more than 4 GPUs + if len(combination) <= 4: + yield self.check_select_gpus, combination def check_select_gpus(self, gpu_list): job_id = self.create_model(select_gpus_list=','.join(gpu_list), batch_size=len(gpu_list)) @@ -418,7 +425,7 @@ def classify_one_for_job(self, job_id, test_misclassification=True): image_path = os.path.join(self.imageset_folder, image_path) with open(image_path, 'rb') as infile: # StringIO wrapping is needed to simulate POST file upload. - image_upload = (StringIO(infile.read()), 'image.png') + image_upload = (BytesIO(infile.read()), 'image.png') rv = self.app.post( '/models/images/classification/classify_one?job_id=%s' % job_id, @@ -457,7 +464,7 @@ def test_retrain(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']), 'should have at least snapshot' options = { @@ -475,7 +482,7 @@ def test_retrain_twice(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']), 'should have at least snapshot' options_2 = { 'method': 'previous', @@ -569,7 +576,7 @@ def test_clone(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content1 = json.loads(rv.data) + content1 = json.loads(rv.get_data(as_text=True)) # Clone job1 as job2 options_2 = { @@ -580,7 +587,7 @@ def test_clone(self): assert self.model_wait_completion(job2_id) == 'Done', 'second job failed' rv = self.app.get('/models/%s/json' % job2_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content2 = json.loads(rv.data) + content2 = json.loads(rv.get_data(as_text=True)) # These will be different content1.pop('id') @@ -629,7 +636,7 @@ def check_download(self, extension): def test_index_json(self): rv = self.app.get('/index/json') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) found = False for m in content['models']: if m['id'] == self.model_id: @@ -640,7 +647,7 @@ def test_index_json(self): def test_model_json(self): rv = self.app.get('/models/%s/json' % self.model_id) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert content['id'] == self.model_id, 'id %s != %s' % (content['id'], self.model_id) assert content['dataset_id'] == self.dataset_id, 'dataset_id %s != %s' % ( content['dataset_id'], self.dataset_id) @@ -662,12 +669,12 @@ def test_edit_notes(self): def test_classify_one(self): # test first image in first category - category = self.imageset_paths.keys()[0] + category = list(self.imageset_paths.keys())[0] image_path = self.imageset_paths[category][0] image_path = os.path.join(self.imageset_folder, image_path) with open(image_path, 'rb') as infile: # StringIO wrapping is needed to simulate POST file upload. - image_upload = (StringIO(infile.read()), 'image.png') + image_upload = (BytesIO(infile.read()), 'image.png') rv = self.app.post( '/models/images/classification/classify_one?job_id=%s' % self.model_id, @@ -685,12 +692,12 @@ def test_classify_one(self): def test_classify_one_json(self): # test last image in last category - category = self.imageset_paths.keys()[-1] + category = list(self.imageset_paths.keys())[-1] image_path = self.imageset_paths[category][-1] image_path = os.path.join(self.imageset_folder, image_path) with open(image_path, 'rb') as infile: # StringIO wrapping is needed to simulate POST file upload. - image_upload = (StringIO(infile.read()), 'image.png') + image_upload = (BytesIO(infile.read()), 'image.png') rv = self.app.post( '/models/images/classification/classify_one/json?job_id=%s' % self.model_id, @@ -700,13 +707,13 @@ def test_classify_one_json(self): } ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) assert data['predictions'][0][0] == category, 'image misclassified' def test_classify_many(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image image_path = os.path.join(self.imageset_folder, image_path) @@ -714,7 +721,7 @@ def test_classify_many(self): label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/classify_many?job_id=%s' % self.model_id, @@ -727,14 +734,14 @@ def test_classify_many(self): def test_classify_many_from_folder(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image textfile_images += '%s %d\n' % (image_path, label_id) label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/classify_many?job_id=%s' % self.model_id, @@ -748,7 +755,7 @@ def test_classify_many_from_folder(self): def test_classify_many_invalid_ground_truth(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image image_path = os.path.join(self.imageset_folder, image_path) @@ -757,7 +764,7 @@ def test_classify_many_invalid_ground_truth(self): label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/classify_many?job_id=%s' % self.model_id, @@ -770,7 +777,7 @@ def test_classify_many_invalid_ground_truth(self): def test_classify_many_json(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image image_path = os.path.join(self.imageset_folder, image_path) @@ -778,14 +785,14 @@ def test_classify_many_json(self): label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/classify_many/json?job_id=%s' % self.model_id, data={'image_list': file_upload} ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) assert 'classifications' in data, 'invalid response' # verify classification of first image in each category for category in self.imageset_paths.keys(): @@ -797,7 +804,7 @@ def test_classify_many_json(self): def test_top_n(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image image_path = os.path.join(self.imageset_folder, image_path) @@ -805,7 +812,7 @@ def test_top_n(self): label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/top_n?job_id=%s' % self.model_id, @@ -816,19 +823,19 @@ def test_top_n(self): assert rv.status_code == 200, 'POST failed with %s\n\n%s' % (rv.status_code, body) keys = self.imageset_paths.keys() for key in keys: - assert key in rv.data, '"%s" not found in the response' + assert key in rv.get_data(as_text=True), '"%s" not found in the response' def test_top_n_from_folder(self): textfile_images = '' label_id = 0 - for label, images in self.imageset_paths.iteritems(): + for label, images in self.imageset_paths.items(): for image in images: image_path = image textfile_images += '%s %d\n' % (image_path, label_id) label_id += 1 # StringIO wrapping is needed to simulate POST file upload. - file_upload = (StringIO(textfile_images), 'images.txt') + file_upload = (BytesIO(textfile_images.encode('utf-8')), 'images.txt') rv = self.app.post( '/models/images/classification/top_n?job_id=%s' % self.model_id, @@ -840,7 +847,7 @@ def test_top_n_from_folder(self): assert rv.status_code == 200, 'POST failed with %s\n\n%s' % (rv.status_code, body) keys = self.imageset_paths.keys() for key in keys: - assert key in rv.data, '"%s" not found in the response' + assert key in rv.get_data(as_text=True), '"%s" not found in the response' def test_inference_while_training(self): # make sure we can do inference while all GPUs are in use for training @@ -862,7 +869,7 @@ def test_inference_while_training(self): image_path = os.path.join(self.imageset_folder, image_path) with open(image_path, 'rb') as infile: # StringIO wrapping is needed to simulate POST file upload. - image_upload = (StringIO(infile.read()), 'image.png') + image_upload = (BytesIO(infile.read()), 'image.png') # create a long-running training job job2_id = self.create_model( @@ -884,7 +891,7 @@ def test_inference_while_training(self): '/models/images/classification/classify_one/json?job_id=%s' % self.model_id, data={'image_file': image_upload} ) - json.loads(rv.data) + json.loads(rv.get_data(as_text=True)) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code finally: self.delete_model(job2_id) @@ -1125,7 +1132,7 @@ class TestCaffeCreatedCropInNetwork(BaseTestCreatedCropInNetwork, test_utils.Caf @unittest.skipIf( - not CaffeFramework().can_accumulate_gradients(), + not config_value('caffe')['loaded'] or not CaffeFramework().can_accumulate_gradients(), 'This version of Caffe cannot accumulate gradients') class TestBatchAccumulationCaffe(BaseViewsTestWithDataset, test_utils.CaffeMixin): TRAIN_EPOCHS = 1 @@ -1220,7 +1227,7 @@ class TestTorchCreatedWideMultiStepLR(BaseTestCreatedWide, test_utils.TorchMixin class TestTorchLeNet(BaseTestCreated, test_utils.TorchMixin): IMAGE_WIDTH = 28 IMAGE_HEIGHT = 28 - TRAIN_EPOCHS = 20 + TRAIN_EPOCHS = 40 # standard lenet model will adjust to color # or grayscale images @@ -1340,7 +1347,7 @@ def test_python_layer(self): assert self.model_wait_completion(job_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']), 'should have at least snapshot' diff --git a/digits/model/images/classification/views.py b/digits/model/images/classification/views.py index 4216a4b65..1533d622d 100644 --- a/digits/model/images/classification/views.py +++ b/digits/model/images/classification/views.py @@ -23,6 +23,7 @@ from digits.utils.routing import request_wants_json, job_from_request from digits.webapp import scheduler +from digits.job import Job blueprint = flask.Blueprint(__name__, __name__) """ @@ -40,7 +41,7 @@ def read_image_list(image_list, image_folder, num_test_images): continue # might contain a numerical label at the end - match = re.match(r'(.*\S)\s+(\d+)$', line) + match = re.match(r'(.*\S)\s+(\d+)$', line.decode('utf-8')) if match: path = match.group(1) ground_truth = int(match.group(2)) @@ -131,7 +132,7 @@ def create(): add_learning_rate = len(form.learning_rate.data) > 1 # Add swept batch_size - sweeps = [dict(s.items() + [('batch_size', bs)]) for bs in form.batch_size.data for s in sweeps[:]] + sweeps = [{**s, **{'batch_size': bs}} for bs in form.batch_size.data for s in sweeps[:]] add_batch_size = len(form.batch_size.data) > 1 n_jobs = len(sweeps) @@ -309,6 +310,7 @@ def create(): rms_decay=form.rms_decay.data, shuffle=form.shuffle.data, data_aug=data_aug, + blob_format=form.nvcaffe_blob_format.data, ) ) @@ -586,7 +588,7 @@ def classify_many(): top1_accuracy = round(100.0 * n_top1_accurate / n_ground_truth, 2) top5_accuracy = round(100.0 * n_top5_accurate / n_ground_truth, 2) per_class_accuracy = [] - for x in xrange(n_labels): + for x in range(n_labels): n_examples = sum(confusion_matrix[x]) per_class_accuracy.append( round(100.0 * confusion_matrix[x, x] / n_examples, 2) if n_examples > 0 else None) @@ -689,9 +691,9 @@ def top_n(): images_per_category = min(top_n, len(images)) # Can't have more categories than the number of labels or the number of outputs n_categories = min(indices.shape[1], len(labels)) - for i in xrange(n_categories): + for i in range(n_categories): result_images = [] - for j in xrange(images_per_category): + for j in range(images_per_category): result_images.append(images[indices[j][i]]) results.append(( labels[i], @@ -712,7 +714,7 @@ def get_datasets(): return [(j.id(), j.name()) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, ImageClassificationDatasetJob) and (j.status.is_running() or j.status == Status.DONE)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -732,7 +734,7 @@ def get_default_standard_network(): def get_previous_networks(): return [(j.id(), j.name()) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, ImageClassificationModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -740,7 +742,7 @@ def get_previous_networks(): def get_previous_networks_fulldetails(): return [(j) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, ImageClassificationModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -760,7 +762,7 @@ def get_previous_network_snapshots(): def get_pretrained_networks(): return [(j.id(), j.name()) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, PretrainedModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -768,6 +770,6 @@ def get_pretrained_networks(): def get_pretrained_networks_fulldetails(): return [(j) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, PretrainedModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] diff --git a/digits/model/images/generic/test_views.py b/digits/model/images/generic/test_views.py index 6fb5ca530..4bd72ea40 100644 --- a/digits/model/images/generic/test_views.py +++ b/digits/model/images/generic/test_views.py @@ -14,7 +14,7 @@ try: from cStringIO import StringIO except ImportError: - from StringIO import StringIO + from io import StringIO from bs4 import BeautifulSoup @@ -212,9 +212,8 @@ def create_model(cls, learning_rate=None, **kwargs): if request_json: if rv.status_code != 200: - print json.loads(rv.data) raise RuntimeError('Model creation failed with %s' % rv.status_code) - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) if 'jobs' in data.keys(): return [j['id'] for j in data['jobs']] else: @@ -222,13 +221,13 @@ def create_model(cls, learning_rate=None, **kwargs): # expect a redirect if not 300 <= rv.status_code <= 310: - print 'Status code:', rv.status_code + print('Status code:', rv.status_code) s = BeautifulSoup(rv.data, 'html.parser') div = s.select('div.alert-danger') if div: - print div[0] + print(div[0]) else: - print rv.data + print(rv.get_data(as_text=True)) raise RuntimeError('Failed to create dataset - status %s' % rv.status_code) job_id = cls.job_id_from_response(rv) @@ -268,7 +267,7 @@ class BaseTestViews(BaseViewsTest): def test_page_model_new(self): rv = self.app.get('/models/images/generic/new') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - assert 'New Image Model' in rv.data, 'unexpected page format' + assert 'New Image Model' in rv.get_data(as_text=True), 'unexpected page format' def test_nonexistent_model(self): assert not self.model_exists('foo'), "model shouldn't exist" @@ -324,7 +323,7 @@ def test_snapshot_interval_2(self): assert self.model_wait_completion(job_id) == 'Done', 'create failed' rv = self.app.get('/models/%s/json' % job_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']) > 1, 'should take >1 snapshot' def test_snapshot_interval_0_5(self): @@ -332,7 +331,7 @@ def test_snapshot_interval_0_5(self): assert self.model_wait_completion(job_id) == 'Done', 'create failed' rv = self.app.get('/models/%s/json' % job_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']) == 2, 'should take 2 snapshots' @unittest.skipIf( @@ -364,9 +363,11 @@ def check_select_gpu(self, gpu_index): def test_select_gpus(self): # test all possible combinations gpu_list = config_value('gpu_list').split(',') - for i in xrange(len(gpu_list)): + for i in range(len(gpu_list)): for combination in itertools.combinations(gpu_list, i + 1): - yield self.check_select_gpus, combination + # Don't test more than 4 GPUs + if len(combination) <= 4: + yield self.check_select_gpus, combination def check_select_gpus(self, gpu_list): job_id = self.create_model(select_gpus_list=','.join(gpu_list), batch_size=len(gpu_list)) @@ -413,7 +414,7 @@ def test_retrain(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']), 'should have at least snapshot' options = { @@ -431,7 +432,7 @@ def test_retrain_twice(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert len(content['snapshots']), 'should have at least snapshot' options_2 = { 'method': 'previous', @@ -485,7 +486,7 @@ def test_clone(self): assert self.model_wait_completion(job1_id) == 'Done', 'first job failed' rv = self.app.get('/models/%s/json' % job1_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content1 = json.loads(rv.data) + content1 = json.loads(rv.get_data(as_text=True)) # Clone job1 as job2 options_2 = { @@ -496,7 +497,7 @@ def test_clone(self): assert self.model_wait_completion(job2_id) == 'Done', 'second job failed' rv = self.app.get('/models/%s/json' % job2_id) assert rv.status_code == 200, 'json load failed with %s' % rv.status_code - content2 = json.loads(rv.data) + content2 = json.loads(rv.get_data(as_text=True)) # These will be different content1.pop('id') @@ -546,7 +547,7 @@ def check_download(self, extension): def test_index_json(self): rv = self.app.get('/index/json') assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) found = False for m in content['models']: if m['id'] == self.model_id: @@ -557,7 +558,7 @@ def test_index_json(self): def test_model_json(self): rv = self.app.get('/models/%s/json' % self.model_id) assert rv.status_code == 200, 'page load failed with %s' % rv.status_code - content = json.loads(rv.data) + content = json.loads(rv.get_data(as_text=True)) assert content['id'] == self.model_id, 'expected different job_id' assert len(content['snapshots']) > 0, 'no snapshots in list' @@ -605,7 +606,7 @@ def test_infer_one_json(self): } ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) assert data['outputs']['output'][0][0] > 0 and \ data['outputs']['output'][0][1] > 0, \ 'image regression result is wrong: %s' % data['outputs']['output'] @@ -673,7 +674,7 @@ def test_infer_many_json(self): data={'image_list': file_upload} ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) assert 'outputs' in data, 'invalid response' def test_infer_db_json(self): @@ -684,7 +685,7 @@ def test_infer_db_json(self): data={'db_path': self.val_db_path} ) assert rv.status_code == 200, 'POST failed with %s\n\n%s' % (rv.status_code, rv.data) - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) assert 'outputs' in data, 'invalid response' @@ -733,7 +734,7 @@ def test_infer_extension_json(self): } ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) output = data['outputs'][data['outputs'].keys()[0]]['output'] assert output[0] > 0 and \ output[1] < 0, \ @@ -828,7 +829,7 @@ def test_infer_one_json(self): data={'image_file': image_upload} ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) data_shape = np.array(data['outputs']['output']).shape if not self.VARIABLE_SIZE_DATASET: if data_shape != (1, self.CHANNELS, self.IMAGE_WIDTH, self.IMAGE_HEIGHT): @@ -865,7 +866,7 @@ def test_infer_one_noresize_json(self): data={'image_file': image_upload, 'dont_resize': 'y'} ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) data_shape = np.array(data['outputs']['output']).shape if data_shape != (1,) + shape: raise ValueError("Shapes differ: got %s expected %s" % (repr(data_shape), repr((1,) + shape))) @@ -1225,7 +1226,7 @@ def test_infer_one_json(self): ) assert rv.status_code == 200, 'POST failed with %s' % rv.status_code # make sure the shape of the output matches the shape of the input - data = json.loads(rv.data) + data = json.loads(rv.get_data(as_text=True)) output = np.array(data['outputs']['output'][0]) assert output.shape == (1, self.CROP_SIZE, self.CROP_SIZE), \ 'shape mismatch: %s' % str(output.shape) diff --git a/digits/model/images/generic/views.py b/digits/model/images/generic/views.py index 4e7d11ada..0a1a85f4a 100644 --- a/digits/model/images/generic/views.py +++ b/digits/model/images/generic/views.py @@ -21,6 +21,7 @@ from digits.utils.forms import fill_form_if_cloned, save_form_to_job from digits.utils.routing import request_wants_json, job_from_request from digits.webapp import scheduler +from digits.job import Job blueprint = flask.Blueprint(__name__, __name__) @@ -104,7 +105,7 @@ def create(extension_id=None): add_learning_rate = len(form.learning_rate.data) > 1 # Add swept batch_size - sweeps = [dict(s.items() + [('batch_size', bs)]) for bs in form.batch_size.data for s in sweeps[:]] + sweeps = [{**s, **{'batch_size': bs}} for bs in form.batch_size.data for s in sweeps[:]] add_batch_size = len(form.batch_size.data) > 1 n_jobs = len(sweeps) @@ -269,6 +270,7 @@ def create(extension_id=None): rms_decay=form.rms_decay.data, shuffle=form.shuffle.data, data_aug=data_aug, + blob_format=form.nvcaffe_blob_format.data, ) ) @@ -784,7 +786,7 @@ def get_datasets(extension_id): if (isinstance(j, GenericImageDatasetJob) or isinstance(j, GenericDatasetJob)) and (j.status.is_running() or j.status == Status.DONE)] return [(j.id(), j.name()) - for j in sorted(jobs, cmp=lambda x, y: cmp(y.id(), x.id()))] + for j in sorted(jobs, key=Job.id)] def get_inference_visualizations(dataset, inputs, outputs): @@ -810,7 +812,7 @@ def get_inference_visualizations(dataset, inputs, outputs): visualizations = [] # process data n = len(inputs['ids']) - for idx in xrange(n): + for idx in range(n): input_id = inputs['ids'][idx] input_data = inputs['data'][idx] output_data = {key: outputs[key][idx] for key in outputs} @@ -831,7 +833,7 @@ def get_inference_visualizations(dataset, inputs, outputs): def get_previous_networks(): return [(j.id(), j.name()) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, GenericImageModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -839,7 +841,7 @@ def get_previous_networks(): def get_previous_networks_fulldetails(): return [(j) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, GenericImageModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -859,7 +861,7 @@ def get_previous_network_snapshots(): def get_pretrained_networks(): return [(j.id(), j.name()) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, PretrainedModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] @@ -867,7 +869,7 @@ def get_pretrained_networks(): def get_pretrained_networks_fulldetails(): return [(j) for j in sorted( [j for j in scheduler.jobs.values() if isinstance(j, PretrainedModelJob)], - cmp=lambda x, y: cmp(y.id(), x.id()) + key=Job.id ) ] From b0e6b6884d58424134ec7ed9f7a202ef49eb12b0 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 13:58:56 -0400 Subject: [PATCH 11/21] Update for Python3 --- digits/model/forms.py | 14 +++++++++++++ digits/model/tasks/__init__.py | 9 +++++---- digits/model/tasks/caffe_train.py | 20 +++++++++++-------- digits/model/tasks/tensorflow_train.py | 4 ++-- .../model/tasks/test_caffe_sanity_checks.py | 4 +++- digits/model/tasks/torch_train.py | 2 +- digits/model/tasks/train.py | 3 ++- digits/model/views.py | 4 ++-- 8 files changed, 41 insertions(+), 19 deletions(-) diff --git a/digits/model/forms.py b/digits/model/forms.py index b6e205728..cc739661c 100644 --- a/digits/model/forms.py +++ b/digits/model/forms.py @@ -159,6 +159,20 @@ def validate_py_ext(form, field): "need a bigger batch size for training but it doesn't fit in memory).") ) + nvcaffe_blob_format = utils.forms.SelectField( + 'Blob format', + choices=[ + ('NVCaffe', 'NVCaffe'), + ('Compatible', 'Compatible'), + ], + tooltip=("Newer NVCaffe stores blobs in a more efficient, BVLC-Caffe-incompatible, format. " + "Selecting 'Compatible' makes .caffemodel files compatible with BVLC Caffe. " + "Ignored in BVLC or other frameworks. " + "Users have to remove 'store_blob_in_old_format' line in solver.prototxt " + "when it is reused in BVLC Caffe"), + default='NVCaffe' + ) + # Solver types solver_type = utils.forms.SelectField( diff --git a/digits/model/tasks/__init__.py b/digits/model/tasks/__init__.py index 1a4ac7a8e..653998154 100644 --- a/digits/model/tasks/__init__.py +++ b/digits/model/tasks/__init__.py @@ -1,18 +1,19 @@ # Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. from __future__ import absolute_import -from .caffe_train import CaffeTrainTask -from .torch_train import TorchTrainTask from .train import TrainTask __all__ = [ - 'CaffeTrainTask', - 'TorchTrainTask', 'TrainTask', ] from digits.config import config_value # noqa +if config_value('caffe')['loaded']: + from .caffe_train import CaffeTrainTask + __all__.append('CaffeTrainTask') + if config_value('tensorflow')['enabled']: from .tensorflow_train import TensorflowTrainTask # noqa __all__.append('TensorflowTrainTask') + diff --git a/digits/model/tasks/caffe_train.py b/digits/model/tasks/caffe_train.py index 80ff8f8b6..373ff40d8 100644 --- a/digits/model/tasks/caffe_train.py +++ b/digits/model/tasks/caffe_train.py @@ -13,7 +13,7 @@ from google.protobuf import text_format import numpy as np import platform -import scipy +from skimage import transform from .train import TrainTask import digits @@ -230,6 +230,7 @@ def before_run(self): def get_mean_image(self, mean_file, resize=False): mean_image = None + spline_order = 1 with open(self.dataset.path(mean_file), 'rb') as f: blob = caffe_pb2.BlobProto() blob.MergeFromString(f.read()) @@ -264,10 +265,11 @@ def get_mean_image(self, mean_file, resize=False): # other than 3 or 4. If it's 1, imresize expects an # array. if (len(shape) == 2 or (len(shape) == 3 and (shape[2] == 3 or shape[2] == 4))): - mean_image = scipy.misc.imresize(mean_image, (data_shape[2], data_shape[3])) + mean_image = transform.resize(mean_image, (data_shape[2], data_shape[3]), + order=spline_order, preserve_range=True).astype(np.uint8) else: - mean_image = scipy.misc.imresize(mean_image[:, :, 0], - (data_shape[2], data_shape[3])) + mean_image = transform.resize(mean_image[:, :, 0],(data_shape[2], data_shape[3]), + order=spline_order, preserve_range=True).astype(np.uint8) mean_image = np.expand_dims(mean_image, axis=2) mean_image = mean_image.transpose(2, 0, 1) mean_image = mean_image.astype('float') @@ -510,6 +512,8 @@ def save_files_classification(self): solver.solver_type = getattr(solver, self.solver_type) solver.net = self.train_val_file + if hasattr(solver, 'store_blobs_in_old_format') and self.blob_format is not None: + solver.store_blobs_in_old_format = True if self.blob_format=='Compatible' else False # Set CPU/GPU mode if config_value('caffe')['cuda_enabled'] and \ bool(config_value('gpu_list')): @@ -1139,7 +1143,7 @@ def after_runtime_error(self): # return the last 20 lines self.traceback = '\n'.join(lines[len(lines) - 20:]) if 'DIGITS_MODE_TEST' in os.environ: - print output + print(output) # TrainTask overrides @@ -1412,7 +1416,7 @@ def get_layer_statistics(self, data): y, x = np.histogram(data, bins=20) y = list(y.astype(np.float32)) ticks = x[[0, len(x) / 2, -1]] - x = [((x[i] + x[i + 1]) / 2.0).astype(np.float32) for i in xrange(len(x) - 1)] + x = [((x[i] + x[i + 1]) / 2.0).astype(np.float32) for i in range(len(x) - 1)] ticks = list(ticks.astype(np.float32)) return (mean, std, [y, x, ticks]) @@ -1459,7 +1463,7 @@ def infer_many_images(self, data_shape = (constants.DEFAULT_BATCH_SIZE,) + data_shape outputs = None - for chunk in [caffe_images[x:x + data_shape[0]] for x in xrange(0, len(caffe_images), data_shape[0])]: + for chunk in [caffe_images[x:x + data_shape[0]] for x in range(0, len(caffe_images), data_shape[0])]: new_shape = (len(chunk),) + data_shape[1:] if net.blobs['data'].data.shape != new_shape: net.blobs['data'].reshape(*new_shape) @@ -1479,7 +1483,7 @@ def infer_many_images(self, else: for name, blob in output.iteritems(): outputs[name] = np.vstack((outputs[name], blob)) - print 'Processed %s/%s images' % (len(outputs[outputs.keys()[0]]), len(caffe_images)) + print('Processed %s/%s images' % (len(outputs[outputs.keys()[0]]), len(caffe_images))) return outputs diff --git a/digits/model/tasks/tensorflow_train.py b/digits/model/tasks/tensorflow_train.py index 876d3075f..046223466 100644 --- a/digits/model/tasks/tensorflow_train.py +++ b/digits/model/tasks/tensorflow_train.py @@ -459,7 +459,7 @@ def after_runtime_error(self): self.traceback = traceback if 'DIGITS_MODE_TEST' in os.environ: - print output + print(output) @override def detect_timeline_traces(self): @@ -732,7 +732,7 @@ def get_layer_statistics(self, data): y, x = np.histogram(data, bins=20) y = list(y) ticks = x[[0, len(x)/2, -1]] - x = [(x[i]+x[i+1])/2.0 for i in xrange(len(x)-1)] + x = [(x[i]+x[i+1])/2.0 for i in range(len(x)-1)] ticks = list(ticks) return (mean, std, [y, x, ticks]) diff --git a/digits/model/tasks/test_caffe_sanity_checks.py b/digits/model/tasks/test_caffe_sanity_checks.py index 8666f6fa4..bd317953d 100644 --- a/digits/model/tasks/test_caffe_sanity_checks.py +++ b/digits/model/tasks/test_caffe_sanity_checks.py @@ -1,10 +1,12 @@ # Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. from __future__ import absolute_import +from digits import test_utils +test_utils.skipIfNotFramework('caffe') + from .caffe_train import CaffeTrainTask, CaffeTrainSanityCheckError from google.protobuf import text_format -from digits import test_utils # Must import after importing digit.config import caffe_pb2 diff --git a/digits/model/tasks/torch_train.py b/digits/model/tasks/torch_train.py index 64cedfa25..7d6b47def 100644 --- a/digits/model/tasks/torch_train.py +++ b/digits/model/tasks/torch_train.py @@ -753,7 +753,7 @@ def get_layer_statistics(self, data): y, x = np.histogram(data, bins=20) y = list(y) ticks = x[[0, len(x) / 2, -1]] - x = [(x[i] + x[i + 1]) / 2.0 for i in xrange(len(x) - 1)] + x = [(x[i] + x[i + 1]) / 2.0 for i in range(len(x) - 1)] ticks = list(ticks) return (mean, std, [y, x, ticks]) diff --git a/digits/model/tasks/train.py b/digits/model/tasks/train.py index b8d9eea00..561f381e0 100644 --- a/digits/model/tasks/train.py +++ b/digits/model/tasks/train.py @@ -65,6 +65,7 @@ def __init__(self, job, dataset, train_epochs, snapshot_interval, learning_rate, self.network = kwargs.pop('network', None) self.framework_id = kwargs.pop('framework_id', None) self.data_aug = kwargs.pop('data_aug', None) + self.blob_format = kwargs.pop('blob_format', None) super(TrainTask, self).__init__(job_dir=job.dir(), **kwargs) self.pickver_task_train = PICKLE_VERSION @@ -365,7 +366,7 @@ def save_output(self, d, name, kind, value): d[name].data.append(value) else: # we might have missed one - for _ in xrange(epoch_len - name_len - 1): + for _ in range(epoch_len - name_len - 1): d[name].data.append(None) d[name].data.append(value) diff --git a/digits/model/views.py b/digits/model/views.py index 27f1c6bc2..e5c892071 100644 --- a/digits/model/views.py +++ b/digits/model/views.py @@ -185,7 +185,7 @@ def visualize_lr(): datalist = [] for j, lr in enumerate(lrs): data = ['Learning Rate %d' % j] - for i in xrange(101): + for i in range(101): if policy == 'fixed': data.append(lr) elif policy == 'step': @@ -354,7 +354,7 @@ def download(job_id, extension): # and store in tempfile (for archive) info = json.dumps(job.json_dict(verbose=False, epoch=epoch), sort_keys=True, indent=4, separators=(',', ': ')) info_io = io.BytesIO() - info_io.write(info) + info_io.write(info.encode('utf-8')) b = io.BytesIO() if extension in ['tar', 'tar.gz', 'tgz', 'tar.bz2']: From c2d49360503b3aa476e1d233df6a41dee62952a4 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 14:00:55 -0400 Subject: [PATCH 12/21] Update for Python3 From 763cf93d2bef97a8bb3af1460d611a52a62a8e4c Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 14:05:45 -0400 Subject: [PATCH 13/21] Update for Python3 --- digits/templates/datasets/generic/show.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/digits/templates/datasets/generic/show.html b/digits/templates/datasets/generic/show.html index 705ea022e..1d66703ab 100644 --- a/digits/templates/datasets/generic/show.html +++ b/digits/templates/datasets/generic/show.html @@ -30,7 +30,7 @@

{{task.name()}}

{{ task.label_shape }}
- {% for name,path in task.dbs.iteritems() %} + {% for name,path in task.dbs.items() %}
{{ name }} DB
{{ job.path(path) }}
From d55a75de22aa2cde8494e01f04ea795ad36c928a Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 14:06:47 -0400 Subject: [PATCH 14/21] Update for Python3 From 8ab498f626f5fe80e4910cd701a117423bc5b196 Mon Sep 17 00:00:00 2001 From: danielwen002 Date: Fri, 23 Aug 2019 14:09:14 -0400 Subject: [PATCH 15/21] Update for Python3 --- digits/templates/helper.html | 8 ++++---- digits/templates/layout.html | 2 +- digits/templates/socketio.html | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/digits/templates/helper.html b/digits/templates/helper.html index 8ecbf966f..47ce4b1de 100644 --- a/digits/templates/helper.html +++ b/digits/templates/helper.html @@ -15,7 +15,7 @@ {% macro print_errors(form) %} {% if form.errors %}
- {% for field_name, field in form._fields.iteritems() %} + {% for field_name, field in form._fields.items() %} {% if field.errors %}
    • @@ -30,7 +30,7 @@ {% endif %} {% if form.warnings %}
      - {% for field_name, field in form._fields.iteritems() %} + {% for field_name, field in form._fields.items() %} {% if field.warnings %}
      • @@ -48,7 +48,7 @@ {% macro print_combined_errors(errors, warnings) %} {% if errors %}
        - {% for field_name, field in errors.iteritems() %} + {% for field_name, field in errors.items() %}
        • {% for e in field %} @@ -61,7 +61,7 @@ {% endif %} {% if warnings %}
          - {% for field_name, field in warnings.iteritems() %} + {% for field_name, field in warnings.items() %}
          • {% for e in field %} diff --git a/digits/templates/layout.html b/digits/templates/layout.html index d7dc3b2a8..3792124a8 100644 --- a/digits/templates/layout.html +++ b/digits/templates/layout.html @@ -96,7 +96,7 @@ window.onload = function () { $('.autocomplete_path').autocomplete({ - serviceUrl: '/autocomplete/path', + serviceUrl: "{{config['URL_PREFIX']}}/autocomplete/path", formatResult: function (suggestion, currentValue) { function baseName(str) diff --git a/digits/templates/socketio.html b/digits/templates/socketio.html index a59dc131f..bd1cf3d41 100644 --- a/digits/templates/socketio.html +++ b/digits/templates/socketio.html @@ -4,7 +4,7 @@