Fix (#28): serialization of indexed Pandas.DataFrames

Author: maciejlach

CHANGELOG.txt

@@ -1,3 +1,9 @@
+------------------------------------------------------------------------------
+  qPython 1.0.1 [2015.08.11]
+------------------------------------------------------------------------------
+
+  - Fix: serialization of indexed Pandas.DataFrames 
+
 ------------------------------------------------------------------------------
   qPython 1.0.0 [2015.04.10]
 ------------------------------------------------------------------------------

conda.recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: qpython
-  version: 1.0.0
+  version: 1.0.1
 
 build:
     number: 1

doc/source/conf.py

@@ -64,7 +64,7 @@ def __getattr__(cls, name):
 
 # General information about the project.
 project = u'qPython'
-copyright = u'2014, DEVnet'
+copyright = u'2014-2015, DEVnet'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -73,7 +73,7 @@ def __getattr__(cls, name):
 # The short X.Y version.
 version = '1.0'
 # The full version, including alpha/beta/rc tags.
-release = '1.0-beta'
+release = '1.0.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

qpython/__init__.py

@@ -17,7 +17,15 @@
 __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']
 
 
-__version__ = '1.0.0'
+__version__ = '1.0.1'
+
+try:
+    from qpython.fastutils import uncompress
+except:
+    __is_cython_enabled__ = False
+else:
+    __is_cython_enabled__ = True
+
 
 class MetaData(object):
     '''Utility class for enriching data structures with meta data, e.g. qtype hint.'''

qpython/_pandas.py

@@ -146,7 +146,7 @@ def _write_pandas_series(self, data, qtype = None):
 
         if qtype is None:
             # determinate type based on first element of the numpy array
-            qtype = Q_TYPE.get(type(data[0]), QGENERAL_LIST)
+            qtype = Q_TYPE.get(type(data.iloc[0]), QGENERAL_LIST)
 
             if qtype == QSTRING:
                 # assume we have a generic list of strings -> force representation as symbol list

samples/console.py

@@ -14,11 +14,13 @@
 #  limitations under the License.
 #
 
+import qpython
 from qpython import qconnection
 from qpython.qtype import QException
 
 
 if __name__ == '__main__':
+    print('qPython %s Cython extensions enabled: %s' % (qpython.__version__, qpython.__is_cython_enabled__))
     with qconnection.QConnection(host = 'localhost', port = 5000) as q:
         print(q)
         print('IPC version: %s. Is connected: %s' % (q.protocol_version, q.is_connected()))

setup.py

@@ -38,7 +38,7 @@ def read(fname):
 
 
 setup(name = 'qPython',
-      version = '1.0.0',
+      version = '1.0.1',
       description = 'kdb+ interfacing library for Python',
       long_description=read('README.rst'),
 

tests/QExpressions3.out

@@ -212,6 +212,8 @@ flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker's Guide
 6200630B0002000000706F73006461746573000000020000000B00030000006431006432006433000E00030000006E0100007900000000000080
 ([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))
 636200630B000100000065696400000001000000070003000000E903000000000000EA03000000000000EB030000000000006200630B0002000000706F73006461746573000000020000000B00030000006431006432006433000E00030000006E0100007900000000000080
+([k: 1 2 3] v: `a`b`c)
+636200630b00010000006b000000010000000700030000000100000000000000020000000000000003000000000000006200630b000100000076000000010000000b0003000000610062006300
 0Ng
 fe00000000000000000000000000000000
 "G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"

tests/pandas_test.py

@@ -156,6 +156,11 @@
                                                                                                ),
                                                                       'meta': MetaData(**{'qtype': QKEYED_TABLE, 'pos': QSYMBOL_LIST, 'dates': QDATE_LIST, 'eid': QLONG_LIST}),
                                                                       'index': ['eid'] }),
+                     ('([k: 1 2 3] v: `a`b`c)',
+                                                                     {'data': pandas.DataFrame({'k':[1,2,3],'v':['a','b','c']}),
+                                                                      'meta': MetaData(**{'qtype': QKEYED_TABLE}),
+                                                                      'index': ['k'],
+                                                                      'compare_meta': False }),
                                     ))
 
 
@@ -261,7 +266,8 @@ def test_reading_pandas():
                         result = result.reset_index()
                         result.meta = meta
 
-                    assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta)
+                    if not 'compare_meta' in value or value['compare_meta']:
+                        assert value['meta'].as_dict() == result.meta.as_dict(), 'deserialization failed qtype: %s, expected: %s actual: %s' % (query, value['meta'], result.meta)
                     assert compare(value['data'], result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value['data'], result)
                 else:
                     assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result)