Skip to content

Commit 78ee80a

Browse files
mgckindmgckind
committed
Merge pull request #70 from mgckind/test
Added some unitests
2 parents 9afce6c + f1976d8 commit 78ee80a

File tree

5 files changed

+548
-3
lines changed

5 files changed

+548
-3
lines changed

‎easyaccess/easyaccess.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -2426,10 +2426,10 @@ def ea_import(self, import_line='', help=False):
24262426

24272427
if help:
24282428
self.do_help_function('all')
2429-
return
2430-
if import_line != '':
2429+
return True
2430+
if import_line != '':
24312431
self.do_import(' '+import_line)
2432-
return
2432+
return True
24332433

24342434

24352435

‎tests/test_api.py

+320
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,320 @@
1+
import unittest
2+
import easyaccess as ea
3+
import numpy as np
4+
import pandas as pd
5+
import os
6+
import fitsio
7+
8+
def create_test_data():
9+
r = np.linspace(0,360,100)
10+
d = np.linspace(-90,90,100)
11+
ra,dec = np.meshgrid(r,d)
12+
dtype = [('RA',float),('DEC',float)]
13+
return np.rec.fromarrays([ra.flat,dec.flat],dtype=dtype)
14+
15+
class TestApi(unittest.TestCase):
16+
17+
con = ea.connect(quiet=True)
18+
tablename = 'testtable'
19+
nrows = 10000
20+
prefetch = 4000
21+
chunk = 1000
22+
sqlfile = 'temp.sql'
23+
csvfile = 'temp.csv'
24+
fitsfile = 'temp.fits'
25+
h5file = 'temp.h5'
26+
27+
28+
def test_ea_import(self):
29+
test1 = self.con.ea_import('wrapped')
30+
if test1 is not None:
31+
self.assertTrue(test1)
32+
test2 = self.con.ea_import('wrapped', help=True)
33+
if test2 is not None:
34+
self.assertTrue(test2)
35+
36+
def test_pandas_to_db(self):
37+
data = create_test_data()
38+
df = pd.DataFrame(data)
39+
self.assertEqual( len(df), self.nrows)
40+
try:
41+
self.con.drop_table(self.tablename)
42+
except:
43+
pass
44+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
45+
cursor = self.con.cursor()
46+
self.assertTrue(self.con.ping())
47+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
48+
fetch = temp.fetchall()
49+
self.assertEqual(len(fetch), self.nrows)
50+
# appending
51+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename, append=True))
52+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
53+
fetch = temp.fetchall()
54+
self.assertEqual(len(fetch), self.nrows*2)
55+
self.con.drop_table(self.tablename)
56+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename, append=True))
57+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
58+
fetch = temp.fetchall()
59+
self.assertEqual(len(fetch), self.nrows)
60+
self.con.drop_table(self.tablename)
61+
cursor.close()
62+
63+
def test_query_to_pandas(self):
64+
data = create_test_data()
65+
df = pd.DataFrame(data)
66+
self.assertEqual( len(df), self.nrows)
67+
try:
68+
self.con.drop_table(self.tablename)
69+
except:
70+
pass
71+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
72+
query = 'select RA,DEC from {:}'.format(self.tablename.upper())
73+
df2 = self.con.query_to_pandas(query)
74+
self.assertEqual( len(df), len(df2))
75+
self.assertEqual( df.columns.values.tolist().sort(), df2.columns.values.tolist().sort())
76+
#iterator
77+
df3 = self.con.query_to_pandas(query, prefetch=4000, iterator=True)
78+
self.assertEqual( len(df3.next()), 4000)
79+
self.assertEqual( df3.next().columns.values.tolist().sort(), df.columns.values.tolist().sort())
80+
self.assertEqual( len(df3.next()), 2000)
81+
self.con.drop_table(self.tablename)
82+
83+
def test_describe_table(self):
84+
data = create_test_data()
85+
df = pd.DataFrame(data)
86+
self.assertEqual( len(df), self.nrows)
87+
try:
88+
self.con.drop_table(self.tablename)
89+
except:
90+
pass
91+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
92+
self.assertEqual(len(self.con.describe_table(self.tablename)), 2)
93+
self.con.drop_table(self.tablename)
94+
95+
def test_loadsql(self):
96+
data = create_test_data()
97+
df = pd.DataFrame(data)
98+
self.assertEqual( len(df), self.nrows)
99+
try:
100+
self.con.drop_table(self.tablename)
101+
except:
102+
pass
103+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
104+
query = """
105+
-- This is a comment
106+
select RA, DEC from %s -- this is another comment
107+
""" % self.tablename
108+
with open(self.sqlfile,'w') as F: F.write(query)
109+
df2 = self.con.query_to_pandas(self.con.loadsql(self.sqlfile))
110+
self.assertEqual( len(df), len(df2))
111+
self.assertEqual( df.columns.values.tolist().sort(), df2.columns.values.tolist().sort())
112+
query = """
113+
-- This is a comment
114+
select RA, DEC from %s ; -- this is another comment
115+
""" % self.tablename
116+
with open(self.sqlfile,'w') as F: F.write(query)
117+
df2 = self.con.query_to_pandas(self.con.loadsql(self.sqlfile))
118+
self.assertEqual( len(df), len(df2))
119+
self.assertEqual( df.columns.values.tolist().sort(), df2.columns.values.tolist().sort())
120+
self.con.drop_table(self.tablename)
121+
os.remove(self.sqlfile)
122+
123+
124+
def test_mytables(self):
125+
df = self.con.mytables()
126+
self.assertTrue('FGOTTENMETADATA' in df['TABLE_NAME'].values.tolist())
127+
128+
129+
def test_load_table_csv(self):
130+
data = create_test_data()
131+
df = pd.DataFrame(data)
132+
self.assertEqual( len(df), self.nrows)
133+
df.to_csv(self.csvfile, index=False, float_format='%.8f', sep=',')
134+
self.assertTrue(os.path.exists(self.csvfile))
135+
self.con.drop_table(os.path.splitext(self.csvfile)[0].upper())
136+
# name from filename
137+
self.assertTrue(self.con.load_table(self.csvfile))
138+
cursor = self.con.cursor()
139+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.csvfile)[0].upper())
140+
fetch = temp.fetchall()
141+
self.assertEqual(len(fetch), self.nrows)
142+
## appending
143+
self.assertTrue(self.con.append_table(self.csvfile))
144+
cursor = self.con.cursor()
145+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.csvfile)[0].upper())
146+
fetch = temp.fetchall()
147+
self.assertEqual(len(fetch), self.nrows*2)
148+
self.con.drop_table(os.path.splitext(self.csvfile)[0].upper())
149+
# name from tablename
150+
self.con.drop_table(self.tablename)
151+
self.assertTrue(self.con.load_table(self.csvfile, name=self.tablename))
152+
cursor = self.con.cursor()
153+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
154+
fetch = temp.fetchall()
155+
self.assertEqual(len(fetch), self.nrows)
156+
## appending
157+
self.assertTrue(self.con.append_table(self.csvfile, name=self.tablename))
158+
cursor = self.con.cursor()
159+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
160+
fetch = temp.fetchall()
161+
self.assertEqual(len(fetch), self.nrows*2)
162+
# chunksize
163+
self.con.drop_table(self.tablename)
164+
self.assertTrue(self.con.load_table(self.csvfile, name=self.tablename, chunksize=self.chunk))
165+
cursor = self.con.cursor()
166+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
167+
fetch = temp.fetchall()
168+
self.assertEqual(len(fetch), self.nrows)
169+
## appending
170+
self.assertTrue(self.con.append_table(self.csvfile, name=self.tablename, chunksize=self.chunk))
171+
cursor = self.con.cursor()
172+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
173+
fetch = temp.fetchall()
174+
self.assertEqual(len(fetch), self.nrows*2)
175+
self.con.drop_table(self.tablename)
176+
os.remove(self.csvfile)
177+
178+
179+
def test_load_table_fits(self):
180+
data = create_test_data()
181+
fitsio.write(self.fitsfile, data, clobber=True)
182+
self.assertTrue(os.path.exists(self.fitsfile))
183+
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
184+
# name from filename
185+
self.assertTrue(self.con.load_table(self.fitsfile))
186+
cursor = self.con.cursor()
187+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.fitsfile)[0].upper())
188+
fetch = temp.fetchall()
189+
self.assertEqual(len(fetch), self.nrows)
190+
## appending
191+
self.assertTrue(self.con.append_table(self.fitsfile))
192+
cursor = self.con.cursor()
193+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.fitsfile)[0].upper())
194+
fetch = temp.fetchall()
195+
self.assertEqual(len(fetch), self.nrows*2)
196+
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
197+
# name from tablename
198+
self.con.drop_table(self.tablename)
199+
self.assertTrue(self.con.load_table(self.fitsfile, name=self.tablename))
200+
cursor = self.con.cursor()
201+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
202+
fetch = temp.fetchall()
203+
self.assertEqual(len(fetch), self.nrows)
204+
## appending
205+
self.assertTrue(self.con.append_table(self.fitsfile, name=self.tablename))
206+
cursor = self.con.cursor()
207+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
208+
fetch = temp.fetchall()
209+
self.assertEqual(len(fetch), self.nrows*2)
210+
# chunksize
211+
self.con.drop_table(self.tablename)
212+
self.assertTrue(self.con.load_table(self.fitsfile, name=self.tablename, chunksize=self.chunk))
213+
cursor = self.con.cursor()
214+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
215+
fetch = temp.fetchall()
216+
self.assertEqual(len(fetch), self.nrows)
217+
## appending
218+
self.assertTrue(self.con.append_table(self.fitsfile, name=self.tablename, chunksize=self.chunk))
219+
cursor = self.con.cursor()
220+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
221+
fetch = temp.fetchall()
222+
self.assertEqual(len(fetch), self.nrows*2)
223+
self.con.drop_table(self.tablename)
224+
os.remove(self.fitsfile)
225+
226+
def test_load_table_hdf5(self):
227+
data = create_test_data()
228+
df = pd.DataFrame(data)
229+
self.assertEqual( len(df), self.nrows)
230+
df.to_hdf(self.h5file, key='data')
231+
self.assertTrue(os.path.exists(self.h5file))
232+
self.con.drop_table(os.path.splitext(self.h5file)[0].upper())
233+
# name from filename
234+
self.assertTrue(self.con.load_table(self.h5file))
235+
cursor = self.con.cursor()
236+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.h5file)[0].upper())
237+
fetch = temp.fetchall()
238+
self.assertEqual(len(fetch), self.nrows)
239+
## appending
240+
self.assertTrue(self.con.append_table(self.h5file))
241+
cursor = self.con.cursor()
242+
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.h5file)[0].upper())
243+
fetch = temp.fetchall()
244+
self.assertEqual(len(fetch), self.nrows*2)
245+
self.con.drop_table(os.path.splitext(self.h5file)[0].upper())
246+
# name from tablename
247+
self.con.drop_table(self.tablename)
248+
self.assertTrue(self.con.load_table(self.h5file, name=self.tablename))
249+
cursor = self.con.cursor()
250+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
251+
fetch = temp.fetchall()
252+
self.assertEqual(len(fetch), self.nrows)
253+
## appending
254+
self.assertTrue(self.con.append_table(self.h5file, name=self.tablename))
255+
cursor = self.con.cursor()
256+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
257+
fetch = temp.fetchall()
258+
self.assertEqual(len(fetch), self.nrows*2)
259+
self.con.drop_table(self.tablename)
260+
os.remove(self.h5file)
261+
262+
263+
def test_query_and_save(self):
264+
data = create_test_data()
265+
df = pd.DataFrame(data)
266+
self.assertEqual( len(df), self.nrows)
267+
cursor = self.con.cursor()
268+
try:
269+
self.con.drop_table(self.tablename)
270+
except:
271+
pass
272+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
273+
query = 'select RA,DEC from %s' % self.tablename.upper()
274+
self.con.query_and_save(query, self.csvfile, print_time=False)
275+
self.assertTrue(os.path.exists(self.csvfile))
276+
self.con.query_and_save(query, self.fitsfile, print_time=False)
277+
self.assertTrue(os.path.exists(self.fitsfile))
278+
self.con.query_and_save(query, self.h5file, print_time=False)
279+
self.assertTrue(os.path.exists(self.h5file))
280+
os.remove(self.csvfile)
281+
os.remove(self.fitsfile)
282+
os.remove(self.h5file)
283+
for i in range(34):
284+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename, append=True))
285+
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
286+
fetch = temp.fetchall()
287+
self.assertEqual(len(fetch), self.nrows*35)
288+
self.con.outfile_max_mb = 1
289+
self.con.query_and_save(query, self.csvfile, print_time=False)
290+
for i in range(6):
291+
self.assertTrue(os.path.exists(os.path.splitext(self.csvfile)[0]+'_00000'+str(i+1)+'.csv'))
292+
os.remove(os.path.splitext(self.csvfile)[0]+'_00000'+str(i+1)+'.csv')
293+
self.con.query_and_save(query, self.fitsfile, print_time=False)
294+
for i in range(4):
295+
self.assertTrue(os.path.exists(os.path.splitext(self.fitsfile)[0]+'_00000'+str(i+1)+'.fits'))
296+
os.remove(os.path.splitext(self.fitsfile)[0]+'_00000'+str(i+1)+'.fits')
297+
298+
self.con.outfile_max_mb = 1000
299+
self.con.drop_table(self.tablename)
300+
301+
def test_inline_functions(self):
302+
data = create_test_data()
303+
df = pd.DataFrame(data)
304+
self.assertEqual( len(df), self.nrows)
305+
cursor = self.con.cursor()
306+
try:
307+
self.con.drop_table(self.tablename)
308+
except:
309+
pass
310+
self.assertTrue(self.con.pandas_to_db(df, tablename=self.tablename))
311+
query = 'select /*p: Y.my_sum(ra,dec) as testcol*/ from %s' % self.tablename
312+
self.con.ea_import('wrapped as Y')
313+
df = self.con.query_to_pandas(query)
314+
self.assertEqual(len(df), self.nrows)
315+
self.assertTrue('TESTCOL' in df.columns.values.tolist())
316+
self.con.drop_table(self.tablename)
317+
318+
319+
if __name__ == '__main__':
320+
unittest.main()

‎tests/test_connection.py

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import unittest
2+
import easyaccess as ea
3+
4+
class TestConnection(unittest.TestCase):
5+
6+
def test_connect_dessci(self):
7+
con = ea.connect('dessci', quiet=True)
8+
self.assertTrue(con.ping())
9+
10+
def test_connect_desoper(self):
11+
con = ea.connect('desoper', quiet=True)
12+
self.assertTrue(con.ping())
13+
14+
def test_connect_destest(self):
15+
con = ea.connect('destest', quiet=True)
16+
self.assertTrue(con.ping())
17+
18+
#@unittest.skip("Not implemented yet")
19+
#def test_connect_memsql(self):
20+
# con = ea.connect('memsql')
21+
# self.assertTrue(con)
22+
23+
if __name__ == '__main__':
24+
unittest.main()
25+
#suite = unittest.TestLoader().loadTestsFromTestCase(TestConnection)
26+
#unittest.TextTestRunner(verbosity=2).run(suite)

0 commit comments

Comments
 (0)