Migrate to flake8 from pycodestyle, and clean up source code.

omniduct/__init__.py

@@ -1,3 +1,5 @@
+# flake8: noqa
+
 from omniduct.duct import Duct
 from omniduct.registry import DuctRegistry
 from . import protocols

omniduct/caches/base.py

@@ -8,10 +8,9 @@
 
 from omniduct.duct import Duct
 from omniduct.utils.config import config
+from omniduct.utils.debug import logger
 from omniduct.utils.docs import quirk_docs
 
-from ..utils.debug import logger
-
 config.register('cache_fail_hard',
                 description='Raise exception if cache fails to save.',
                 default=False)

omniduct/caches/local.py

@@ -6,8 +6,9 @@
 
 import six
 
-from ..utils.storage import ensure_path_exists
-from ..utils.debug import logger
+from omniduct.utils.debug import logger
+from omniduct.utils.storage import ensure_path_exists
+
 from .base import Cache
 
 
@@ -51,7 +52,7 @@ def get_hash(cls, id_str):
         Returns:
             str: The sha1 hash of the id_str.
         """
-        if sys.version_info.major == 3 or sys.version_info.major == 2 and isinstance(id_str, unicode):
+        if sys.version_info.major == 3 or sys.version_info.major == 2 and isinstance(id_str, unicode):  # noqa: F821
             id_str = id_str.encode('utf8')
         return hashlib.sha1(id_str).hexdigest()
 

omniduct/databases/base.py

@@ -9,14 +9,17 @@
 
 import sqlparse
 from decorator import decorator
+from IPython import get_ipython
 from jinja2 import StrictUndefined, Template
 
-from . import cursor_formatters
 from omniduct.caches.base import cached_method
 from omniduct.duct import Duct
 from omniduct.utils.debug import logger, logging_scope
 from omniduct.utils.docs import quirk_docs
-from omniduct.utils.magics import MagicsProvider, process_line_arguments, process_line_cell_arguments
+from omniduct.utils.magics import (MagicsProvider, process_line_arguments,
+                                   process_line_cell_arguments)
+
+from . import cursor_formatters
 
 logging.getLogger('requests').setLevel(logging.WARNING)
 
@@ -156,7 +159,7 @@ def statement_hash(cls, statement, cleanup=True):
         """
         if cleanup:
             statement = cls.statement_cleanup(statement)
-        if sys.version_info.major == 3 or sys.version_info.major == 2 and isinstance(statement, unicode):
+        if sys.version_info.major == 3 or sys.version_info.major == 2 and isinstance(statement, unicode):  # noqa: F821
             statement = statement.encode('utf8')
         return hashlib.sha256(statement).hexdigest()
 
@@ -654,12 +657,12 @@ def query_magic(*args, **kwargs):
 
         @register_line_cell_magic("{}.{}".format(base_name, 'execute'))
         @process_line_cell_arguments
-        def query_magic(*args, **kwargs):
+        def execute_magic(*args, **kwargs):
             return statement_executor_magic('execute', *args, **kwargs)
 
         @register_line_cell_magic("{}.{}".format(base_name, 'stream'))
         @process_line_cell_arguments
-        def query_magic(*args, **kwargs):
+        def stream_magic(*args, **kwargs):
             return statement_executor_magic('stream', *args, **kwargs)
 
         @register_cell_magic("{}.{}".format(base_name, 'template'))

omniduct/databases/cursor_formatters.py

@@ -1,9 +1,11 @@
 import csv
 import io
 import pickle
+from distutils.version import LooseVersion
 
 import pandas as pd
-from distutils.version import LooseVersion
+
+from omniduct.utils.debug import logger
 
 
 class CursorFormatter(object):

omniduct/databases/hiveserver2.py

@@ -1,7 +1,6 @@
 from __future__ import absolute_import
 
 import json
-import logging
 import os
 import re
 import shutil
@@ -11,7 +10,6 @@
 import pandas as pd
 from jinja2 import Template
 
-from omniduct.utils.config import config
 from omniduct.utils.debug import logger
 from omniduct.utils.processes import Timeout, run_in_subprocess
 
@@ -139,7 +137,7 @@ def _execute(self, statement, cursor=None, wait=True, poll_interval=1):
         log_offset = 0
 
         if self.driver == 'pyhive':
-            from TCLIService.ttypes import TOperationState
+            from TCLIService.ttypes import TOperationState  # noqa: F821
             cursor.execute(statement, **{'async': True})
 
             if wait:
@@ -166,6 +164,7 @@ def _cursor_empty(self, cursor):
         return False
 
     def _cursor_wait(self, cursor, poll_interval=1):
+        from TCLIService.ttypes import TOperationState  # noqa: F821
         status = cursor.poll().operationState
         while status in (TOperationState.INITIALIZED_STATE, TOperationState.RUNNING_STATE):
             time.sleep(poll_interval)

omniduct/databases/presto.py

@@ -9,9 +9,9 @@
 import six
 from future.utils import raise_with_traceback
 
+from omniduct._version import __version__
 from omniduct.utils.debug import logger
 
-from .._version import __version__
 from .base import DatabaseClient
 from .schemas import SchemasMixin
 

omniduct/databases/schemas.py

@@ -1,28 +1,27 @@
 from __future__ import absolute_import
-import logging
 
 import pandas as pd
-from sqlalchemy import (ARRAY, Boolean, Column, Float, Integer, MetaData,
-                        String, Table, inspect, types)
+import sqlalchemy
+from sqlalchemy import Table
+from sqlalchemy import types as sql_types
 
 from omniduct.utils.debug import logger
 
-
 try:
     from pyhive.sqlalchemy_presto import PrestoDialect
 
     def get_columns(self, connection, table_name, schema=None, **kw):
         # Extend types supported by PrestoDialect as defined in PyHive
         type_map = {
-            'bigint': types.BigInteger,
-            'integer': types.Integer,
-            'boolean': types.Boolean,
-            'double': types.Float,
-            'varchar': types.String,
-            'timestamp': types.TIMESTAMP,
-            'date': types.DATE,
-            'array<bigint>': ARRAY(Integer),
-            'array<varchar>': ARRAY(String)
+            'bigint': sql_types.BigInteger,
+            'integer': sql_types.Integer,
+            'boolean': sql_types.Boolean,
+            'double': sql_types.Float,
+            'varchar': sql_types.String,
+            'timestamp': sql_types.TIMESTAMP,
+            'date': sql_types.DATE,
+            'array<bigint>': sql_types.ARRAY(sql_types.Integer),
+            'array<varchar>': sql_types.ARRAY(sql_types.String)
         }
 
         rows = self._get_table_columns(connection, table_name, schema)
@@ -32,7 +31,7 @@ def get_columns(self, connection, table_name, schema=None, **kw):
                 coltype = type_map[row.Type]
             except KeyError:
                 logger.warn("Did not recognize type '%s' of column '%s'" % (row.Type, row.Column))
-                coltype = types.NullType
+                coltype = sql_types.NullType
             result.append({
                 'name': row.Column,
                 'type': coltype,
@@ -93,7 +92,7 @@ class Schemas(object):
 
     def __init__(self, metadata):
         self._metadata = metadata
-        self._schema_names = inspect(self._metadata.bind).get_schema_names()
+        self._schema_names = sqlalchemy.inspect(self._metadata.bind).get_schema_names()
         self._schema_cache = {}
 
     def __dir__(self):
@@ -125,7 +124,7 @@ class Schema(object):
     def __init__(self, metadata, schema):
         self._metadata = metadata
         self._schema = schema
-        self._table_names = inspect(self._metadata.bind).get_table_names(schema)
+        self._table_names = sqlalchemy.inspect(self._metadata.bind).get_table_names(schema)
         self._table_cache = {}
 
     def __dir__(self):

omniduct/duct.py

@@ -1,5 +1,4 @@
 import atexit
-import decorator
 import functools
 import getpass
 import inspect
@@ -11,14 +10,15 @@
 from builtins import input
 from enum import Enum
 
+import decorator
 import six
 from future.utils import raise_with_traceback, with_metaclass
 
-from omniduct.errors import DuctServerUnreachable, DuctProtocolUnknown
+from omniduct.errors import DuctProtocolUnknown, DuctServerUnreachable
 from omniduct.utils.debug import logger, logging_scope
 from omniduct.utils.dependencies import check_dependencies
 from omniduct.utils.docs import quirk_docs
-from omniduct.utils.ports import naive_load_balancer, is_port_bound
+from omniduct.utils.ports import is_port_bound, naive_load_balancer
 
 
 class ProtocolRegisteringABCMeta(ABCMeta):

omniduct/filesystems/base.py

@@ -1,8 +1,9 @@
 import io
 from abc import abstractmethod
-from collections import namedtuple, OrderedDict
+from collections import OrderedDict, namedtuple
 
 import pandas as pd
+
 from omniduct.duct import Duct
 from omniduct.utils.docs import quirk_docs
 from omniduct.utils.magics import MagicsProvider, process_line_arguments

omniduct/filesystems/webhdfs_helpers.py

@@ -2,12 +2,12 @@
 import xml.dom.minidom
 
 import requests
+from six.moves import http_client
+
 from pywebhdfs import errors
 from pywebhdfs.webhdfs import (PyWebHdfsClient, _is_standby_exception,
                                _move_active_host_to_head)
 
-from six.moves import http_client
-
 
 class OmniductPyWebHdfsClient(PyWebHdfsClient):
 

omniduct/protocols.py

@@ -1,9 +1,11 @@
+# flake8: noqa
+
 from .caches.local import LocalCache
+from .databases.druid import DruidClient
 from .databases.hiveserver2 import HiveServer2Client
+from .databases.neo4j import Neo4jClient
 from .databases.presto import PrestoClient
 from .databases.sqlalchemy import SQLAlchemyClient
-from .databases.neo4j import Neo4jClient
-from .databases.druid import DruidClient
 from .filesystems.local import LocalFsClient
 from .filesystems.s3 import S3Client
 from .filesystems.webhdfs import WebHdfsClient

omniduct/registry.py

@@ -2,10 +2,10 @@
 import yaml
 
 from omniduct.duct import Duct
+from omniduct.errors import DuctProtocolUnknown
+from omniduct.utils.debug import logger
 from omniduct.utils.magics import MagicsProvider
 from omniduct.utils.proxies import NestedDictObjectProxy
-from omniduct.utils.debug import logger
-from omniduct.errors import DuctProtocolUnknown
 
 
 class DuctRegistry(object):

omniduct/remotes/ssh.py

@@ -7,18 +7,19 @@
 from builtins import input
 from io import open
 
-try:  # Python 3
-    from shlex import quote as escape_path
-except ImportError:  # Python 2.7
-    from pipes import quote as escape_path
-
 import pandas as pd
 
+from omniduct.errors import DuctAuthenticationError
 from omniduct.filesystems.base import FileSystemFileDesc
 from omniduct.remotes.base import RemoteClient
 from omniduct.utils.debug import logger
 from omniduct.utils.processes import run_in_subprocess
-from omniduct.errors import DuctAuthenticationError
+
+try:  # Python 3
+    from shlex import quote as escape_path
+except ImportError:  # Python 2.7
+    from pipes import quote as escape_path
+
 
 SSH_ASKPASS = '{omniduct_dir}/utils/ssh_askpass'.format(omniduct_dir=os.path.dirname(__file__))
 SESSION_SSH_USERNAME = None

omniduct/session.py

@@ -6,13 +6,15 @@
 
 from .utils.config import config
 
+__all__ = ['config', 'registry']
+
+
 # The default console width is too wide for most notebooks, leading to ugly logging message / progress bars. We set this
 # to a more reasonable value for Jupyter Notebooks.
 ip = IPython.get_ipython()
 if ip and ip.__class__.__name__ == "ZMQInteractiveShell":
     os.environ['COLUMNS'] = "80"
 
-__all__ = ['config', 'registry', '__author__', '__author_email__', '__version__']
 
 OMNIDUCT_CONFIG = os.environ.get('OMNIDUCT_CONFIG', None) or os.path.expanduser('~/.omniduct/config')
 

omniduct/utils/debug.py

@@ -4,7 +4,6 @@
 import logging
 import sys
 import time
-import types
 
 import progressbar
 import six

omniduct/utils/proxies.py

@@ -30,7 +30,6 @@ def add_nested(tree, key, value):
 
         def flat_to_nested(tree):
             out = {}
-            children = {}
             for k, v in tree.items():
                 add_nested(out, k, v)
             return out

omniduct/utils/submodules.py

@@ -1,6 +1,6 @@
-import sys
 import importlib
 import pkgutil
+import sys
 
 
 def import_submodules(package_name):

setup.cfg

@@ -1,6 +1,6 @@
 [bdist_wheel]
 universal=1
 
-[pycodestyle]
-ignore = E501,E712,E722,W503,W504
+[flake8]
+ignore = E501,E712,E722,W503,W504,W601,W606
 max-line-length = 160

tox.ini

@@ -7,10 +7,10 @@ envlist =
 [testenv]
 deps=
     mock
-    pycodestyle
+    flake8
     pyfakefs
     pytest
     requests
 commands=
     pytest tests
-    pycodestyle --ignore=E501,E712,E722,W503,W504,W601,W606 omniduct tests
+    flake8 omniduct tests