diff --git a/CHANGELOG b/CHANGELOG index 628e8626..69333cfb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,8 @@ * Dropped unlzw in favor of using the gzip utility * Made antenna inclusion/exclusion more robust in lsl.imaging.data.VisibilityDataSet.get_antenna_subset() * Added support for "long (ionospheric) product names" + * Added support for triggered voltage buffer dump files from OVRO-LWA with lsl.reader.ovro + * Added support for triggered voltage buffer dump files from OVRO-LWA in lsl.reader.ldp 2.1.7 * Updated the LWA-SV SSMIF to 2022/11/15 diff --git a/lsl/data/tests/ovro-test.dat b/lsl/data/tests/ovro-test.dat new file mode 100644 index 00000000..07064672 Binary files /dev/null and b/lsl/data/tests/ovro-test.dat differ diff --git a/lsl/reader/gofast.c b/lsl/reader/gofast.c index 994670f4..65a496fb 100644 --- a/lsl/reader/gofast.c +++ b/lsl/reader/gofast.c @@ -41,6 +41,7 @@ short int tbw4LUT[256][2]; float tbnLUT[256]; float drxLUT[256][2]; float tbfLUT[256][2]; +float ovroLUT[256][2]; static void initLWALUTs(void) { // Look-up table inialization function from the VDIFIO library @@ -63,7 +64,7 @@ static void initLWALUTs(void) { tbnLUT[i] -= ((i&128)<<1); } - // DRX & TBF + // DRX & TBF & OVRO for(i=0; i<256; i++) { for(j=0; j<2; j++) { t = (i >> 4*(1-j)) & 15; @@ -72,6 +73,9 @@ static void initLWALUTs(void) { tbfLUT[i][j] = t; tbfLUT[i][j] -= ((t&8)<<1); + + ovroLUT[i][j] = t; + ovroLUT[i][j] -= ((t&8)<<1); } } } @@ -82,13 +86,14 @@ static void initLWALUTs(void) { */ static PyMethodDef GoFastMethods[] = { - {"read_tbw", (PyCFunction) read_tbw, METH_VARARGS, read_tbw_doc }, - {"read_tbn", (PyCFunction) read_tbn, METH_VARARGS, read_tbn_doc }, - {"read_drx", (PyCFunction) read_drx, METH_VARARGS, read_drx_doc }, - {"read_drspec", (PyCFunction) read_drspec, METH_VARARGS, read_drspec_doc}, - {"read_vdif", (PyCFunction) read_vdif, METH_VARARGS|METH_KEYWORDS, read_vdif_doc }, - {"read_tbf", (PyCFunction) read_tbf, METH_VARARGS, read_tbf_doc }, - {"read_cor", (PyCFunction) read_cor, METH_VARARGS, read_cor_doc }, + {"read_tbw", (PyCFunction) read_tbw, METH_VARARGS, read_tbw_doc }, + {"read_tbn", (PyCFunction) read_tbn, METH_VARARGS, read_tbn_doc }, + {"read_drx", (PyCFunction) read_drx, METH_VARARGS, read_drx_doc }, + {"read_drspec", (PyCFunction) read_drspec, METH_VARARGS, read_drspec_doc }, + {"read_vdif", (PyCFunction) read_vdif, METH_VARARGS|METH_KEYWORDS, read_vdif_doc }, + {"read_tbf", (PyCFunction) read_tbf, METH_VARARGS, read_tbf_doc }, + {"read_cor", (PyCFunction) read_cor, METH_VARARGS, read_cor_doc }, + {"read_ovro_spec", (PyCFunction) read_ovro_spec, METH_VARARGS, read_ovro_spec_doc}, {NULL, NULL, 0, NULL } }; @@ -162,6 +167,7 @@ MOD_INIT(_gofast) { PyList_Append(all, PyString_FromString("read_vdif")); PyList_Append(all, PyString_FromString("read_tbf")); PyList_Append(all, PyString_FromString("read_cor")); + PyList_Append(all, PyString_FromString("read_ovro_spec")); PyList_Append(all, PyString_FromString("SyncError")); PyList_Append(all, PyString_FromString("EOFError")); PyList_Append(all, PyString_FromString("NCHAN_COR")); diff --git a/lsl/reader/ldp.py b/lsl/reader/ldp.py index 2db29e92..c31a14c2 100644 --- a/lsl/reader/ldp.py +++ b/lsl/reader/ldp.py @@ -10,6 +10,7 @@ * DRSpecFile * TBFFile * CORFILE + * OVROFile Also included are the LWA1DataFile, LWASVDataFile, and LWADataFile functions that take a filename and try to determine the correct data format object to @@ -36,7 +37,7 @@ from lsl.common.dp import fS from lsl.common.adp import fC -from lsl.reader import tbw, tbn, drx, drspec, tbf, cor, errors +from lsl.reader import tbw, tbn, drx, drspec, tbf, cor, ovro, errors from lsl.reader.buffer import TBNFrameBuffer, DRXFrameBuffer, TBFFrameBuffer, CORFrameBuffer from lsl.reader.utils import * from lsl.reader.base import FrameTimestamp @@ -49,9 +50,9 @@ telemetry.track_module() -__version__ = '0.4' -__all__ = ['TBWFile', 'TBNFile', 'DRXFile', 'DRSpecFile', 'TBFFile', 'LWA1DataFile', - 'LWASVDataFile', 'LWADataFile'] +__version__ = '0.5' +__all__ = ['TBWFile', 'TBNFile', 'DRXFile', 'DRSpecFile', 'TBFFile', 'OVROFile', + 'LWA1DataFile', 'LWASVDataFile', 'OVROLWADataFile', 'LWADataFile'] class _LDPFileRegistry(object): @@ -2423,6 +2424,305 @@ def LWASVDataFile(filename=None, fh=None, ignore_timetag_errors=False, buffering return ldpInstance +class OVROFile(LDPFileBase): + """ + Class to make it easy to interface with a OVRO-LWA triggered voltage buffer + dump file. Methods defined for this class are: + * get_info - Get information about the file's contents + * get_remaining_frame_count - Get the number of frames remaining in the file + * offset - Offset a specified number of seconds into the file + * read_frame - Read and return a single `lsl.reader.ovro.Frame` instance + * read - Read in the capture and return it as a numpy array + """ + + def _ready_file(self): + """ + Find the start of valid OVRO-LWA triggered voltage dump data. + """ + + self.fh.seek(0) + + return True + + def _describe_file(self): + """ + Describe the OVRO-LWA triggered voltage buffer dump file. + """ + + with FilePositionSaver(self.fh): + # Read in two frames + junkFrame = ovro.read_frame(self.fh) + fmark0 = self.fh.tell() + junkFrame2 = ovro.read_frame(self.fh) + fmark1 = self.fh.tell() + self.fh.seek(-2*(fmark1-fmark0), 1) + header_size = self.fh.tell() + + # Basic file information + try: + filesize = os.fstat(self.fh.fileno()).st_size + except AttributeError: + filesize = self.fh.size + nFramesFile = (filesize - self.fh.tell()) // (fmark1 - fmark0) + srate = 8192 / 196e6 + bits = 4 + nFramesPerObs = ovro.get_frames_per_obs(self.fh) + nchan = ovro.get_channel_count(self.fh) + firstChan = ovro.get_first_channel(self.fh) + + # Find the "real" starttime + start = junkFrame.time + startRaw = junkFrame.header.timetag + + # Antenna parameters + nantenna = junkFrame.payload.data.shape[-2]*junkFrame.payload.data.shape[-1] + + # Jump over the header + self.fh.seek(header_size, 0) + + # Calculate the frequencies + freq = junkFrame.channel_freqs + + self.description = {'size': filesize, 'nframe': nFramesFile, 'frame_size': fmark1-fmark0, + 'header_size': header_size, 'sample_rate': srate, 'data_bits': bits, + 'nantenna': nantenna, 'nchan': nchan, 'freq1': freq, 'start_time': start, + 'start_time_samples': startRaw} + + def get_remaining_frame_count(self): + """ + Return the number of frames left in the file. + """ + + return (self.description['size'] - self.fh.tell()) // self.description['frame_size'] + + def offset(self, offset): + """ + Offset a specified number of seconds in an open OVRO-LWA triggered + voltage buffer dump file. This function returns the exact offset time. + """ + + # Find out where we really are taking into account the buffering + buffer_offset = 0 + if getattr(self, "_timetag", None) is not None: + frame = ovro.read_frame(self.fh) + self.fh.seek(-self.description['frame_size'], 1) + curr = frame.header.time_tag + buffer_offset = curr - self._timetag + buffer_offset = buffer_offset / fS + + offset = offset - buffer_offset + frameOffset = int(offset * self.description['sample_rate']) + frameOffset = int(1.0 * frameOffse) + self.fh.seek(frameOffset*self.description['frame_size'], 1) + + # Update the file metadata + self._describe_file() + + # Reset the timetag checker + self._timetag = None + + return 1.0 * frameOffset / self.description['sample_rate'] + + def read_frame(self): + """ + Read and return a single `lsl.reader.ovro.Frame` instance. + """ + + # Reset the timetag checker + self._timetag = None + + return ovro.read_frame(self.fh) + + def read(self, duration=None, time_in_samples=False): + """ + Read and return the entire OVRO-LWA triggered voltage buffer dump. This + function returns a three-element tuple with elements of: + 0) the actual duration of data read in, + 1) the time tag for the first sample, and + 2) a 3-D Numpy array of data. + + The time tag is returned as seconds since the UNIX epoch as a + `lsl.reader.base.FrameTimestamp` instance by default. However, the time + tags can be returns as samples at `lsl.common.dp.fS` if the + `time_in_samples' keyword is set. + + The sorting order of the output data array is by + digitizer number - 1. + """ + + # Make sure there is file left to read + try: + curr_size = os.fstat(self.fh.fileno()).st_size + except AttributeError: + curr_size = self.fh.size + if self.fh.tell() == curr_size: + raise errors.EOFError() + + # Covert the sample rate to an expected timetag skip + timetagSkip = int(1.0 / self.description['sample_rate'] * fS) + + # Setup the counter variables: frame count and time tag count + if getattr(self, "_timetag", None) is None: + self._timetag = 0 + + # Find out how many frames to read in + if duration is None: + duration = self.description['nframe'] / self.description['sample_rate'] + frame_count = int(round(1.0 * duration * self.description['sample_rate'])) + frame_count = frame_count if frame_count else 1 + duration = frame_count / self.description['sample_rate'] + + nFrameSets = 0 + eofFound = False + setTime = None + data = numpy.zeros((self.description['nantenna'], self.description['nchan'], frame_count), dtype=numpy.complex64) + while True: + if eofFound or nFrameSets == frame_count: + break + + try: + cFrame = ovro.read_frame(self.fh, verbose=False) + except errors.EOFError: + eofFound = True + cFrame = None + + # Continue adding frames if nothing comes out. + if cFrame is None: + continue + + # If something comes out, add it to the data array + cTimetag = cFrame.header.timetag + if self._timetag == 0: + self._timetag = cTimetag - timetagSkip + if cTimetag != self._timetag+timetagSkip: + actStep = cTimetag - self._timetag + if self.ignore_timetag_errors: + warnings.warn(colorfy("{{%%yellow Invalid timetag skip encountered, expected %i, but found %i}}" % (timetagSkip, actStep)), RuntimeWarning) + else: + raise RuntimeError("Invalid timetag skip encountered, expected %i, but found %i" % (timetagSkip, actStep)) + self._timetag = cFrame.header.timetag + + if setTime is None: + if time_in_samples: + setTime = cFrame.header.timetag + else: + setTime = cFrame.time + + subData = cFrame.payload.data + subData.shape = (self.description['nchan'], self.description['nantenna']) + subData = subData.T + + data[:,:,nFrameSets] = subData + nFrameSets += 1 + + # Sanity check at the end to see if we actually read anything. + # This is needed because of how TBF and DRX interact where TBF + # files can be padded at the end with DRX data + if nFrameSets == 0 and duration > 0: + raise errors.EOFError() + + # Adjust the duration to account for all of the things that could + # have gone wrong while reading the data + duration = nFrameSets / self.description['sample_rate'] + + return duration, setTime, data + + +def OVROLWADataFile(filename=None, fh=None, ignore_timetag_errors=False, buffering=-1): + """ + Wrapper around the various LWA-SV-related classes defined here that takes + a file, determines the data type, and initializes and returns the + appropriate LDP class. + """ + + # Open the file as appropriate + is_splitfile = False + if fh is None: + fh = open(filename, 'rb') + else: + filename = fh.name + if not isinstance(fh, SplitFileWrapper): + if fh.mode.find('b') == -1: + fh.close() + fh = open(filename, 'rb') + else: + is_splitfile = True + + # Read a bit of data to try to find the right type + for mode in (drx, ovro): + ## Set if we find a valid frame marker + foundMatch = False + ## Set if we can read more than one valid successfully + foundMode = False + + ## Sort out the frame size. This is tricky because DR spectrometer files + ## have frames of different sizes depending on the mode + if mode == drspec: + try: + mfs = drspec.get_frame_size(fh) + except: + mfs = 0 + else: + mfs = mode.FRAME_SIZE + + ## Loop over the frame size to try and find what looks like valid data. If + ## is is found, set 'foundMatch' to True. + for i in range(mfs): + try: + junkFrame = mode.read_frame(fh) + foundMatch = True + break + except errors.EOFError: + break + except errors.SyncError: + fh.seek(-mfs+1, 1) + + ## Did we strike upon a valid frame? + if foundMatch: + ### Is so, we now need to try and read more frames to make sure we have + ### the correct type of file + fh.seek(-mfs, 1) + + try: + for i in range(2): + junkFrame = mode.read_frame(fh) + foundMode = True + except errors.EOFError: + break + except errors.SyncError: + ### Reset for the next mode... + fh.seek(0) + else: + ### Reset for the next mode... + fh.seek(0) + + ## Did we read more than one valid frame? + if foundMode: + break + + fh.seek(0) + if not is_splitfile: + fh.close() + fh = None + + # Raise an error if nothing is found + if not foundMode: + raise RuntimeError("File '%s' does not appear to be a valid OVRO-LWA data file" % filename) + + # Otherwise, build and return the correct LDPFileBase sub-class + if mode == drx: + ldpInstance = DRXFile(filename=filename, fh=fh, + ignore_timetag_errors=ignore_timetag_errors, + buffering=buffering) + else: + ldpInstance = OVROFile(filename=filename, fh=fh, + ignore_timetag_errors=ignore_timetag_errors, + buffering=buffering) + + # Done + return ldpInstance + + def LWADataFile(filename=None, fh=None, ignore_timetag_errors=False, buffering=-1): """ Wrapper around the various classes defined here that takes a file, @@ -2452,6 +2752,16 @@ def LWADataFile(filename=None, fh=None, ignore_timetag_errors=False, buffering=- except RuntimeError: pass + # OVRO-LWA? + if not found: + try: + ldpInstance = OVROLWADataFile(filename=filename, fh=fh, + ignore_timetag_errors=ignore_timetag_errors, + buffering=buffering) + found = True + except RuntimeError: + pass + # Failed? if not found: raise RuntimeError("File '%s' does not appear to be a valid LWA1 or LWA-SV data file" % filename) diff --git a/lsl/reader/ovro.c b/lsl/reader/ovro.c new file mode 100644 index 00000000..3343676a --- /dev/null +++ b/lsl/reader/ovro.c @@ -0,0 +1,90 @@ +#include "Python.h" +#include +#include +#include +#include + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL gofast_ARRAY_API +#include "numpy/arrayobject.h" + +#include "readers.h" + +/* + OVRO-LWA triggered voltage buffer dump file data section reader +*/ + +PyObject *ovro_method = NULL; +PyObject *ovro_size = NULL; + + +PyObject *read_ovro_spec(PyObject *self, PyObject *args) { + PyObject *ph, *buffer, *output; + PyArrayObject *data; + long i; + int nchan, nstand, npol; + + if(!PyArg_ParseTuple(args, "Oiii", &ph, &nchan, &nstand, &npol)) { + PyErr_Format(PyExc_RuntimeError, "Invalid parameters"); + return NULL; + } + + // Create the output data array + npy_intp dims[3]; + // 4+4-bit Data -> N samples in the data section as nchan channels, nstand stands, and npol pols. + dims[0] = (npy_intp) nchan; + dims[1] = (npy_intp) nstand; + dims[2] = (npy_intp) npol; + data = (PyArrayObject*) PyArray_ZEROS(3, dims, NPY_COMPLEX64, 0); + if(data == NULL) { + PyErr_Format(PyExc_MemoryError, "Cannot create output array"); + Py_XDECREF(data); + return NULL; + } + + // Read from the file + if( ovro_method == NULL ) { + ovro_method = Py_BuildValue("s", "read"); + ovro_size = Py_BuildValue("i", nchan*nstand*npol*1); + } + buffer = PyObject_CallMethodObjArgs(ph, ovro_method, ovro_size, NULL); + if( buffer == NULL ) { + if( PyObject_HasAttrString(ph, "read") ) { + PyErr_Format(PyExc_IOError, "An error occured while reading from the file"); + } else { + PyErr_Format(PyExc_AttributeError, "Object does not have a read() method"); + } + Py_XDECREF(data); + return NULL; + } else if( PyString_GET_SIZE(buffer) != (nchan*nstand*npol*1) ) { + PyErr_Format(EOFError, "End of file encountered during filehandle read"); + Py_XDECREF(data); + Py_XDECREF(buffer); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + + // Fill the data array + const float *fp; + float complex *a; + a = (float complex *) PyArray_DATA(data); + for(i=0; i= 9: + divis = 1e9 + units = 'GHz' + elif scale >= 6: + divis = 1e6 + units = 'MHz' + elif scale >= 3: + divis = 1e3 + units = 'kHz' + else: + divis = 1 + units = 'Hz' + + # Convert the frequency + newFreq = freq / divis + + # Return units and freq + return (newFreq, units) + + +def main(args): + fh = open(args.filename, 'rb') + + # Read in the first frame and get the date/time of the first sample + # of the frame. This is needed to get the list of stands. + junkFrame = ovro.read_frame(fh) + fh.seek(0) + beginDate = junkFrame.time.datetime + + # Figure out how many frames are in the file + nFrames = (os.path.getsize(args.filename) - 1048576) // junkFrame.payload.data.size + + # Make a dummy list of antennas + nstand = junkFrame.payload.data.shape[-2] + npol = junkFrame.payload.data.shape[-1] + antpols = nstand*npol + antennas = list(range(antpols)) + + # Figure out how many frames there are per observation and the number of + # channels that are in the file + nFramesPerObs = ovro.get_frames_per_obs(fh) + nchannels = ovro.get_channel_count(fh) + + # Figure out how many chunks we need to work with + nChunks = nFrames // nFramesPerObs + + # Calculate the frequencies + freq = junkFrame.channel_freqs + + # File summary + print("Filename: %s" % args.filename) + print("Date of First Frame: %s" % str(beginDate)) + print("Frames per Observation: %i" % nFramesPerObs) + print("Channel Count: %i" % nchannels) + print("Frames: %i" % nFrames) + print("===") + print("Chunks: %i" % nChunks) + + spec = numpy.zeros((nchannels,nstand,npol)) + norm = numpy.zeros_like(spec) + for i in range(nChunks): + # Read in the next frame and anticipate any problems that could occur + try: + cFrame = ovro.read_frame(fh) + except errors.EOFError: + break + spec[:,:,:] += numpy.abs(cFrame.payload.data)**2 + norm[:,:,:] += 1 + + spec /= norm + fh.close() + + # Reshape and transpose to get it in to a "normal" order + spec.shape = (spec.shape[0], spec.shape[1]*spec.shape[2]) + spec = spec.T + + # Put the frequencies in the best units possible + freq, units = _best_freq_units(freq) + + # Deal with the `keep` options + if args.keep == 'all': + antpolsDisp = int(numpy.ceil(antpols/20)) + js = [i for i in range(antpols)] + else: + antpolsDisp = int(numpy.ceil(len(args.keep)*2/20)) + if antpolsDisp < 1: + antpolsDisp = 1 + + js = [] + for k in args.keep: + for i,ant in enumerate(antennas): + if ant == k: + js.append(i) + + nPlot = len(js) + if nPlot < 16: + if nPlot % 4 == 0 and nPlot != 4: + figsY = 4 + else: + figsY = 2 + figsX = int(numpy.ceil(1.0*nPlot/figsY)) + else: + figsY = 4 + figsX = 4 + figsN = figsX*figsY + for i in range(antpolsDisp): + # Normal plotting + fig = plt.figure() + for k in range(i*figsN, i*figsN+figsN): + try: + j = js[k] + currSpectra = numpy.squeeze( numpy.log10(spec[j,:])*10.0 ) + except IndexError: + break + ax = fig.add_subplot(figsX, figsY, (k%figsN)+1) + ax.plot(freq, currSpectra, label='Dig: %i' % (antennas[j])) + + ax.set_title('Dig: %i' % (antennas[j],)) + ax.set_xlabel('Frequency [%s]' % units) + ax.set_ylabel('P.S.D. [dB/RBW]') + ax.set_ylim([-10, 30]) + + # Save spectra image if requested + if args.output is not None: + base, ext = os.path.splitext(args.output) + outFigure = "%s-%02i%s" % (base, i+1, ext) + fig.savefig(outFigure) + + plt.draw() + + print("RBW: %.4f %s" % ((freq[1]-freq[0]), units)) + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='read in a OVRO-LWA triggered voltage buffer dump file and create a collection of time-averaged spectra', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('filename', type=str, + help='filename to process') + parser.add_argument('-q', '--quiet', dest='verbose', action='store_false', + help='run %(prog)s in silent mode') + parser.add_argument('-k', '--keep', type=aph.csv_int_list, default='all', + help='only display the following comma-seperated list of stands') + parser.add_argument('-o', '--output', type=str, + help='output file name for spectra image') + args = parser.parse_args() + main(args) + diff --git a/setup.py b/setup.py index 06f20b27..769aef92 100644 --- a/setup.py +++ b/setup.py @@ -314,7 +314,7 @@ def finalize_options(self, *args, **kwargs): # Create the list of extension modules. We do this here so that we can turn # off the DRSU direct module for non-linux system -ExtensionModules = [Extension('reader._gofast', ['lsl/reader/gofast.c', 'lsl/reader/tbw.c', 'lsl/reader/tbn.c', 'lsl/reader/drx.c', 'lsl/reader/drspec.c', 'lsl/reader/vdif.c', 'lsl/reader/tbf.c', 'lsl/reader/cor.c'], include_dirs=[numpy.get_include()], extra_compile_args=['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', '-funroll-loops']), +ExtensionModules = [Extension('reader._gofast', ['lsl/reader/gofast.c', 'lsl/reader/tbw.c', 'lsl/reader/tbn.c', 'lsl/reader/drx.c', 'lsl/reader/drspec.c', 'lsl/reader/vdif.c', 'lsl/reader/tbf.c', 'lsl/reader/cor.c', 'lsl/reader/ovro.c'], include_dirs=[numpy.get_include()], extra_compile_args=['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', '-funroll-loops']), Extension('common._fir', ['lsl/common/fir.cpp'], include_dirs=[numpy.get_include()], libraries=['m'], extra_compile_args=coreExtraFlags, extra_link_args=coreExtraLibs), Extension('correlator._spec', ['lsl/correlator/spec.cpp'], include_dirs=[numpy.get_include()], libraries=['m'], extra_compile_args=coreExtraFlags, extra_link_args=coreExtraLibs), Extension('correlator._stokes', ['lsl/correlator/stokes.cpp'], include_dirs=[numpy.get_include()], libraries=['m'], extra_compile_args=coreExtraFlags, extra_link_args=coreExtraLibs), diff --git a/tests/test_ldp.py b/tests/test_ldp.py index ea3c0adb..e7eca58b 100644 --- a/tests/test_ldp.py +++ b/tests/test_ldp.py @@ -19,7 +19,7 @@ from lsl.reader.utils import SplitFileWrapper -__version__ = "0.2" +__version__ = "0.3" __author__ = "Jayce Dowell" @@ -30,6 +30,8 @@ tbfFile = os.path.join(DATA_BUILD, 'tests', 'tbf-test.dat') +ovroFile = os.path.join(DATA_BUILD, 'tests', 'ovro-test.dat') + class ldp_tests(unittest.TestCase): """A unittest.TestCase collection of unit tests for the lsl.reader @@ -555,6 +557,11 @@ def test_ldp_discover_tbf(self): # TBF self.assertRaises(RuntimeError, ldp.LWA1DataFile, tbfFile) + def test_ldp_discover_ovro(self): + """Test the LDP LWA1DataFile function of OVRO-LWA triggered voltage buffer dump.""" + # TBF + self.assertRaises(RuntimeError, ldp.LWA1DataFile, ovroFile) + ### SplitFileWrapper ### def testldp_splitfilewrapper(self): diff --git a/tests/test_ldp_adp.py b/tests/test_ldp_adp.py index 3c010a1a..8835e7b9 100644 --- a/tests/test_ldp_adp.py +++ b/tests/test_ldp_adp.py @@ -17,7 +17,7 @@ from lsl.reader.utils import SplitFileWrapper -__version__ = "0.1" +__version__ = "0.2" __author__ = "Jayce Dowell" @@ -29,6 +29,8 @@ tbfFile = os.path.join(DATA_BUILD, 'tests', 'tbf-test.dat') corFile = os.path.join(DATA_BUILD, 'tests', 'cor-test.dat') +ovroFile = os.path.join(DATA_BUILD, 'tests', 'ovro-test.dat') + class ldp_adp_tests(unittest.TestCase): """A unittest.TestCase collection of unit tests for the lsl.reader @@ -192,6 +194,11 @@ def test_ldp_discover_cor(self): f = ldp.LWASVDataFile(corFile) self.assertEqual(type(f), ldp.CORFile) + def test_ldp_discover_ovro(self): + """Test the LDP LWASVDataFile function of OVRO-LWA triggered voltage buffer dump.""" + # TBF + self.assertRaises(RuntimeError, ldp.LWASVDataFile, ovroFile) + def test_ldp_discover_all_tbw(self): """Test the LDP LWADataFile function of TBW.""" # TBW diff --git a/tests/test_ldp_ovro.py b/tests/test_ldp_ovro.py new file mode 100644 index 00000000..d973c931 --- /dev/null +++ b/tests/test_ldp_ovro.py @@ -0,0 +1,111 @@ +""" +Unit test for the OVRO-LWA portion of the lsl.reader.ldp module. +""" + +# Python2 compatibility +from __future__ import print_function, division, absolute_import +import sys +if sys.version_info < (3,): + range = xrange + +import os +import unittest + +from lsl.common.paths import DATA_BUILD +from lsl.reader import ldp +from lsl.reader import errors +from lsl.reader.utils import SplitFileWrapper + + +__version__ = "0.1" +__author__ = "Jayce Dowell" + + +tbwFile = os.path.join(DATA_BUILD, 'tests', 'tbw-test.dat') +tbnFile = os.path.join(DATA_BUILD, 'tests', 'tbn-test.dat') +drxFile = os.path.join(DATA_BUILD, 'tests', 'drx-test.dat') +drspecFile = os.path.join(DATA_BUILD, 'tests', 'drspec-test.dat') + +tbfFile = os.path.join(DATA_BUILD, 'tests', 'tbf-test.dat') +corFile = os.path.join(DATA_BUILD, 'tests', 'cor-test.dat') + +ovroFile = os.path.join(DATA_BUILD, 'tests', 'ovro-test.dat') + + +class ldp_ovro_tests(unittest.TestCase): + """A unittest.TestCase collection of unit tests for the lsl.reader + modules.""" + + ### Triggered voltage buffer dump ### + + def test_ldp_ovro(self): + """Test the LDP interface for a triggered voltage buffer dump.""" + + f = ldp.OVROFile(ovroFile) + + # File info + self.assertEqual(f.get_info("sample_rate"), 8192/196e6) + self.assertEqual(f.get_info("data_bits"), 4) + self.assertEqual(f.get_info('nframe'), 10) + + self.assertEqual(f.sample_rate, 8192/196e6) + self.assertEqual(f.data_bits, 4) + self.assertEqual(f.nframe, 10) + + # Read a frame + frame = f.read_frame() + + # Get the remaining frame count + self.assertEqual(f.get_remaining_frame_count(), f.get_info('nframe')-1) + self.assertEqual(f.nframe_remaining, f.get_info('nframe')-1) + + # Reset + f.reset() + + # Read a chunk - short + tInt, tStart, data = f.read(0.1) + + # Close it out + f.close() + + def test_ldp_ovro_nocheck(self): + """Test the LDP interface for a triggered voltage buffer dump.""" + + f = ldp.OVROFile(ovroFile, ignore_timetag_errors=True) + + # File info + self.assertEqual(f.get_info("sample_rate"), 8192/196e6) + self.assertEqual(f.get_info("data_bits"), 4) + self.assertEqual(f.get_info('nframe'), 10) + + self.assertEqual(f.sample_rate, 8192/196e6) + self.assertEqual(f.data_bits, 4) + self.assertEqual(f.nframe, 10) + + # Read a frame + frame = f.read_frame() + + # Get the remaining frame count + self.assertEqual(f.get_remaining_frame_count(), f.get_info('nframe')-1) + self.assertEqual(f.nframe_remaining, f.get_info('nframe')-1) + + # Reset + f.reset() + + # Close it out + f.close() + + +class ldp_ovro_test_suite(unittest.TestSuite): + """A unittest.TestSuite class which contains all of the lsl.reader.ldp + unit tests.""" + + def __init__(self): + unittest.TestSuite.__init__(self) + + loader = unittest.TestLoader() + self.addTests(loader.loadTestsFromTestCase(ldp_ovro_tests)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_reader_ovro.py b/tests/test_reader_ovro.py new file mode 100644 index 00000000..918993eb --- /dev/null +++ b/tests/test_reader_ovro.py @@ -0,0 +1,130 @@ +""" +Unit tests for the lsl.reader modules that are for OVRO-LWA. +""" + +# Python2 compatibility +from __future__ import print_function, division, absolute_import +import sys +if sys.version_info < (3,): + range = xrange + +import os +import numpy +import unittest + +from lsl.common.paths import DATA_BUILD +from lsl.reader import ovro +from lsl.reader import errors + + +__version__ = "0.1" +__author__ = "Jayce Dowell" + + +ovroFile = os.path.join(DATA_BUILD, 'tests', 'ovro-test.dat') + + +class reader_ovro_tests(unittest.TestCase): + """A unittest.TestCase collection of unit tests for the lsl.reader + modules.""" + + ### Triggered voltage buffer dump ### + + def test_ovro_read(self): + """Test reading in a frame from a OVRO-LWA triggered voltage dump file.""" + + fh = open(ovroFile, 'rb') + # First frame stores the first channel + frame1 = ovro.read_frame(fh) + self.assertEqual(frame1.header.first_chan, 2288) + # Second frame + frame2 = ovro.read_frame(fh) + self.assertEqual(frame2.header.first_chan, 2288) + fh.close() + + def test_ovro_errors(self): + """Test OVRO-LWA triggered voltage dump reading errors.""" + + fh = open(ovroFile, 'rb') + # Frames 1 through 10 + for i in range(1,11): + frame = ovro.read_frame(fh) + + # Last frame should be an error (errors.EOFError) + self.assertRaises(errors.EOFError, ovro.read_frame, fh) + fh.close() + + def test_ovro_comps(self): + """Test the OVRO-LWA triggered voltage dump frame comparison operators (>, <, etc.) for time tags.""" + + fh = open(ovroFile, 'rb') + # Frames 1 through 10 + frames = [] + for i in range(1,11): + frames.append(ovro.read_frame(fh)) + fh.close() + + self.assertTrue(0 < frames[0]) + self.assertFalse(0 > frames[0]) + self.assertTrue(frames[-1] >= frames[0]) + self.assertFalse(frames[-1] < frames[0]) + self.assertTrue(frames[0] == frames[0]) + self.assertTrue(frames[0] != frames[-1]) + self.assertFalse(frames[0] != frames[0]) + + def test_ovro_sort(self): + """Test sorting OVRO-LWA triggered voltage dump frames by time tags.""" + + fh = open(ovroFile, 'rb') + # Frames 1 through 9 + frames = [] + for i in range(1,10): + frames.append(ovro.read_frame(fh)) + fh.close() + + frames.sort() + frames = frames[::-1] + + for i in range(1,len(frames)): + self.assertTrue( frames[i-1] >= frames[i] ) + + def test_ovro_math(self): + """Test mathematical operations on OVRO-LWA triggered voltage dump frame data via frames.""" + + fh = open(ovroFile, 'rb') + # Frames 1 through 9 + frames = [] + for i in range(1,10): + frames.append(ovro.read_frame(fh)) + fh.close() + + # Multiplication + frameT = frames[0] * 2.0 + numpy.testing.assert_allclose(frameT.payload.data, 2*frames[0].payload.data, atol=1e-6) + frameT *= 2.0 + numpy.testing.assert_allclose(frameT.payload.data, 4*frames[0].payload.data, atol=1e-6) + frameT = frames[0] * frames[1] + numpy.testing.assert_allclose(frameT.payload.data, frames[0].payload.data*frames[1].payload.data, atol=1e-6) + + # Addition + frameA = frames[0] + 2.0 + numpy.testing.assert_allclose(frameA.payload.data, 2+frames[0].payload.data, atol=1e-6) + frameA += 2.0 + numpy.testing.assert_allclose(frameA.payload.data, 4+frames[0].payload.data, atol=1e-6) + frameA = frames[0] + frames[1] + numpy.testing.assert_allclose(frameA.payload.data, frames[0].payload.data+frames[1].payload.data, atol=1e-6) + + +class reader_ovro_test_suite(unittest.TestSuite): + """A unittest.TestSuite class which contains all of the lsl.reader units + tests.""" + + def __init__(self): + unittest.TestSuite.__init__(self) + + loader = unittest.TestLoader() + self.addTests(loader.loadTestsFromTestCase(reader_ovro_tests)) + + +if __name__ == '__main__': + unittest.main()