Skip to content

Commit 0010f79

Browse files
joannajarmulskajoannajarmulska
committed
Merge branch 'maciejlach-master'
2 parents 2ac86a0 + ce020e7 commit 0010f79

File tree

7 files changed

+175
-104
lines changed

7 files changed

+175
-104
lines changed

‎CHANGELOG.txt

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
------------------------------------------------------------------------------
2+
qPython 1.2.2 [2016.09.21]
3+
------------------------------------------------------------------------------
4+
5+
- Fix: Pandas deserialization of single row tables with string nulls
6+
17
------------------------------------------------------------------------------
28
qPython 1.2.1 [2016.03.29]
39
------------------------------------------------------------------------------
@@ -32,13 +38,13 @@
3238
qPython 1.0.0 [2015.04.10]
3339
------------------------------------------------------------------------------
3440

35-
- Improve type detection for pandas.Series serialization
41+
- Improve type detection for pandas.Series serialization
3642

3743
------------------------------------------------------------------------------
3844
qPython 1.0 RC3 [2015.04.09]
3945
------------------------------------------------------------------------------
4046

41-
- Fix: bug during handshake with blank username/password
47+
- Fix: bug during handshake with blank username/password
4248

4349
------------------------------------------------------------------------------
4450
qPython 1.0 RC2 [2015.01.19]
@@ -58,7 +64,7 @@
5864
------------------------------------------------------------------------------
5965

6066
- Configurable deserialization for temporal vectors
61-
- numpy datetime64/timedelta64 vectors ans atoms are now serializable without
67+
- numpy datetime64/timedelta64 vectors ans atoms are now serializable without
6268
explicit conversions
6369
- QTemporal: null values are now backed as numpy NaT
6470
- Fix: error handling while initializing connection
@@ -79,7 +85,7 @@
7985
qPython 1.0 Beta 4 [2014.07.04]
8086
------------------------------------------------------------------------------
8187

82-
- qtemporallist: force numpy.array type conversion in case of mismatch between
88+
- qtemporallist: force numpy.array type conversion in case of mismatch between
8389
meta.qtype and dtype of raw list
8490
- Enable Travis CI integration
8591
- Update project meta-information

‎qpython/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
__all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection']
1818

1919

20-
__version__ = '1.2.1'
20+
__version__ = '1.2.2'
2121

2222

2323

@@ -66,4 +66,4 @@ def union_dict(self, **kw):
6666
numpy_temporals = False,
6767
pandas = False,
6868
single_char_strings = False
69-
)
69+
)

‎qpython/_pandas.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ def _read_table(self, qtype = QTABLE):
7272
self._buffer.skip() # ignore dict type stamp
7373

7474
columns = self._read_object()
75-
data = self._read_object()
75+
self._buffer.skip() # ignore generic list type indicator
76+
data = QReader._read_general_list(self, qtype)
7677

7778
odict = OrderedDict()
7879
meta = MetaData(qtype = QTABLE)

‎tests/QExpressions3.out

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,4 +225,12 @@ FE8C680A015A495AAB5A65D4BFDDB6A661
225225
"G"$"00000000-0000-0000-0000-000000000000"
226226
FE00000000000000000000000000000000
227227
("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)
228-
0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
228+
0200020000008C680A015A495AAB5A65D4BFDDB6A66100000000000000000000000000000000
229+
1#([] sym:`x`x`x;str:" a")
230+
6200630b000200000073796d00737472000000020000000b000100000078000a000100000020
231+
-1#([] sym:`x`x`x;str:" a")
232+
6200630b000200000073796d00737472000000020000000b000100000078000a000100000061
233+
2#([] sym:`x`x`x`x;str:" aa")
234+
6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000002020
235+
-2#([] sym:`x`x`x`x;str:" aa")
236+
6200630b000200000073796d00737472000000020000000b0002000000780078000a00020000006161

‎tests/pandas_test.py

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
try:
3535
import pandas
3636
BINARY = None
37-
37+
3838
PANDAS_EXPRESSIONS = OrderedDict((
3939
(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)',
4040
{'data': pandas.Series(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), numpy.nan])),
@@ -49,7 +49,7 @@
4949
[b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']),
5050
(b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")',
5151
[b'quick', numpy.nan, b'fox', b'jumps', b'over', b'a lazy', b'dog']),
52-
52+
5353
(b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)),
5454
'meta': MetaData(qtype = QBOOL_LIST) }),
5555
(b'(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)),
@@ -78,7 +78,7 @@
7878
'meta': MetaData(qtype = QDOUBLE_LIST) }),
7979
(b'3.23 0n', {'data': pandas.Series(numpy.array([3.23, numpy.nan])),
8080
'meta': MetaData(qtype = QDOUBLE_LIST) }),
81-
81+
8282
(b'(2001.01m; 0Nm)', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]')),
8383
'meta': MetaData(qtype = QMONTH_LIST) }),
8484
(b'2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')),
@@ -95,7 +95,7 @@
9595
'meta': MetaData(qtype = QTIMESTAMP_LIST) }),
9696
(b'0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])),
9797
'meta': MetaData(qtype = QTIMESPAN_LIST) }),
98-
98+
9999
(b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST),
100100
qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))),
101101
(b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)],
@@ -121,6 +121,32 @@
121121
))
122122
),
123123
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'grade': QSTRING}) }),
124+
(b'1#([] sym:`x`x`x;str:" a")',
125+
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
126+
('str', pandas.Series([' '], dtype = numpy.str).replace(b' ', numpy.nan)),
127+
))
128+
),
129+
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
130+
'single_char_strings': True}),
131+
(b'-1#([] sym:`x`x`x;str:" a")',
132+
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)),
133+
('str', pandas.Series(['a'], dtype = numpy.str)),
134+
))
135+
),
136+
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}),
137+
'single_char_strings': True}),
138+
(b'2#([] sym:`x`x`x`x;str:" aa")',
139+
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
140+
('str', pandas.Series([' ', ' '], dtype = numpy.str).replace(b' ', numpy.nan)),
141+
))
142+
),
143+
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
144+
(b'-2#([] sym:`x`x`x`x;str:" aa")',
145+
{'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)),
146+
('str', pandas.Series(['a', 'a'], dtype = numpy.str).replace(b' ', numpy.nan)),
147+
))
148+
),
149+
'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}),
124150
(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))',
125151
{'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)),
126152
('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))),
@@ -147,7 +173,6 @@
147173
('iq', pandas.Series(numpy.array([], dtype = numpy.int32)))))
148174
),
149175
'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QINT_LIST}) }),
150-
151176
(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))',
152177
{'data': pandas.DataFrame(OrderedDict((('pos', pandas.Series(numpy.array(['d1', 'd2', 'd3'], dtype = numpy.string_))),
153178
('dates', pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')))))
@@ -276,15 +301,18 @@ def test_writing_pandas():
276301

277302
for query, value in iter(PANDAS_EXPRESSIONS.items()):
278303
sys.stdout.write( '%-75s' % query )
304+
single_char_strings = False
279305
if isinstance(value, dict):
280306
data = value['data']
281307
if 'index' in value:
282308
data = data.reset_index(drop = True)
283309
data = data.set_index(value['index'])
310+
if 'single_char_strings' in value:
311+
single_char_strings = value['single_char_strings']
284312
data.meta = value['meta']
285313
else:
286314
data = value
287-
serialized = binascii.hexlify(w.write(data, 1))[16:].lower()
315+
serialized = binascii.hexlify(w.write(data, 1, single_char_strings = single_char_strings))[16:].lower()
288316
assert serialized == BINARY[query].lower(), 'serialization failed: %s, expected: %s actual: %s' % (value, BINARY[query].lower(), serialized)
289317
sys.stdout.write( '.' )
290318

@@ -312,8 +340,3 @@ def test_writing_pandas():
312340
test_writing_pandas()
313341
except ImportError:
314342
pandas = None
315-
316-
317-
318-
319-

0 commit comments

Comments
 (0)