Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix compatibility with Pandas 2 #63

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix compatibility with Pandas 2 #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions devicely/_compat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""Pandas 1/2 compatibility."""

try:
import importlib.metadata as importlib_metadata
except ModuleNotFoundError:
import importlib_metadata

_pd_version = importlib_metadata.version('pandas')
_pd_major = int(_pd_version.split('.')[0])
_have_pd_2 = _pd_major >= 2
# Pandas 1.5 renamed the `line_terminator` parameter of the `to_csv` method to
# `lineterminator` for consistency, and then Pandas 2 removed the old name.
_to_csv_line_terminator = 'lineterminator' if _have_pd_2 else 'line_terminator'
11 changes: 8 additions & 3 deletions devicely/empatica.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import numpy as np
import pandas as pd

from ._compat import _to_csv_line_terminator


class EmpaticaReader:
"""
Expand Down Expand Up @@ -155,7 +157,8 @@ def _write_signal(self, path, dataframe, signal_name):
[self.sample_freqs[signal_name]] * n_cols])
with open(path, 'w') as file:
np.savetxt(file, meta, fmt='%s', delimiter=', ', newline='\n')
dataframe.to_csv(file, index=None, header=None, line_terminator='\n')
dataframe.to_csv(file, index=None, header=None,
**{_to_csv_line_terminator: '\n'})

def _read_ibi(self, path):
try:
Expand All @@ -179,7 +182,8 @@ def _write_ibi(self, path):
file.write(f"{self.start_times['IBI'].value // 1e9}, IBI\n")
write_df = self.IBI.copy()
write_df.index = (write_df.index - self.start_times['IBI']).values.astype(int) / 1e9
write_df.to_csv(file, header=None, line_terminator='\n')
write_df.to_csv(file, header=None,
**{_to_csv_line_terminator: '\n'})

def _read_tags(self, path):
try:
Expand All @@ -200,7 +204,8 @@ def _read_tags(self, path):
def _write_tags(self, path):
if self.tags is not None:
tags_write_series = self.tags.map(lambda x: x.value / 1e9)
tags_write_series.to_csv(path, header=None, index=None, line_terminator='\n')
tags_write_series.to_csv(path, header=None, index=None,
**{_to_csv_line_terminator: '\n'})

def timeshift(self, shift='random'):
"""
Expand Down
8 changes: 6 additions & 2 deletions devicely/everion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import numpy as np
import pandas as pd

from ._compat import _to_csv_line_terminator


class EverionReader:
"""
Read, timeshift and write data generated by Biovotion Everion.
Expand Down Expand Up @@ -240,7 +243,7 @@ def _convert_single_dataframe(self, dataframe, selected_tags=None):
if selected_tags is not None:
dataframe = dataframe[dataframe['tag'].isin(selected_tags)]

dataframe['time'] = dataframe['time'].map(lambda x: x.value) / 10**9
dataframe['time'] = dataframe['time'].astype(np.int64) / 10**9
timestamps_min_and_count = dataframe.groupby('time').agg(
count_min=pd.NamedAgg(column='count', aggfunc='min'),
count_range=pd.NamedAgg(
Expand Down Expand Up @@ -318,7 +321,8 @@ def _write_single_dataframe(self, dataframe, filepath):
writing_dataframe.loc[quality_col.index, 'values'] += ';' + quality_col
writing_dataframe.drop(columns=['quality'], inplace=True)

writing_dataframe.to_csv(filepath, index=None, line_terminator='\n')
writing_dataframe.to_csv(filepath, index=None,
**{_to_csv_line_terminator: '\n'})

def timeshift(self, shift='random'):
"""
Expand Down
14 changes: 10 additions & 4 deletions devicely/faros.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import pandas as pd
import pyedflib as edf

from ._compat import _to_csv_line_terminator


class FarosReader:
"""
Expand Down Expand Up @@ -220,10 +222,14 @@ def _write_to_directory(self, path):
with open(os.path.join(path, 'meta.json'), 'w') as meta_file:
json.dump(meta, meta_file)

self.ECG.to_csv(os.path.join(path, 'ECG.csv'), index=None, line_terminator='\n')
self.ACC.to_csv(os.path.join(path, 'ACC.csv'), index=None, line_terminator='\n')
self.Marker.to_csv(os.path.join(path, 'Marker.csv'), index=None, line_terminator='\n')
self.HRV.to_csv(os.path.join(path, 'HRV.csv'), index=None, line_terminator='\n')
self.ECG.to_csv(os.path.join(path, 'ECG.csv'), index=None,
**{_to_csv_line_terminator: '\n'})
self.ACC.to_csv(os.path.join(path, 'ACC.csv'), index=None,
**{_to_csv_line_terminator: '\n'})
self.Marker.to_csv(os.path.join(path, 'Marker.csv'), index=None,
**{_to_csv_line_terminator: '\n'})
self.HRV.to_csv(os.path.join(path, 'HRV.csv'), index=None,
**{_to_csv_line_terminator: '\n'})

def timeshift(self, shift='random'):
"""
Expand Down
5 changes: 4 additions & 1 deletion devicely/shimmer_plus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import numpy as np
import pandas as pd

from ._compat import _to_csv_line_terminator


class ShimmerPlusReader:
"""
Expand Down Expand Up @@ -69,7 +71,8 @@ def write(self, path):

with open(path, 'w') as f:
f.write(f'"sep={self.delimiter}"\n')
write_df.to_csv(f, index=False, sep=self.delimiter, line_terminator=f"{self.delimiter}\n")
write_df.to_csv(f, index=False, sep=self.delimiter,
**{_to_csv_line_terminator: f"{self.delimiter}\n"})
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm. This raises a ValueError with Python 3.13.

________________________ ShimmerPlusTestCase.test_write ________________________
self = <test_shimmer.ShimmerPlusTestCase testMethod=test_write>
    def test_write(self):
>       self.reader.write(self.WRITE_PATH)
tests/test_shimmer.py:82: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../BUILDROOT/usr/lib/python3.13/site-packages/devicely/shimmer_plus.py:74: in write
    write_df.to_csv(f, index=False, sep=self.delimiter,
/usr/lib64/python3.13/site-packages/pandas/util/_decorators.py:333: in wrapper
    return func(*args, **kwargs)
/usr/lib64/python3.13/site-packages/pandas/core/generic.py:3964: in to_csv
    return DataFrameRenderer(formatter).to_csv(
/usr/lib64/python3.13/site-packages/pandas/io/formats/format.py:1014: in to_csv
    csv_formatter.save()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <pandas.io.formats.csvs.CSVFormatter object at 0x7f2269a70a50>
    def save(self) -> None:
        """
        Create the writer & save.
        """
        # apply compression and byte/text conversion
        with get_handle(
            self.filepath_or_buffer,
            self.mode,
            encoding=self.encoding,
            errors=self.errors,
            compression=self.compression,
            storage_options=self.storage_options,
        ) as handles:
            # Note: self.encoding is irrelevant here
>           self.writer = csvlib.writer(
                handles.handle,
                lineterminator=self.lineterminator,
                delimiter=self.sep,
                quoting=self.quoting,
                doublequote=self.doublequote,
                escapechar=self.escapechar,
                quotechar=self.quotechar,
            )
E           ValueError: bad delimiter or lineterminator value
/usr/lib64/python3.13/site-packages/pandas/io/formats/csvs.py:260: ValueError

So far, I haven't got a clue as to why?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That seems unrelated to this change. The existing call (slightly) abuses lineterminator to always insert a trailing separator. This must have been made more strict in Python 3.13, and I see a few changelog entries that could be related, most likely "gh-113796: Add more validation checks in the csv.Dialect constructor. ValueError is now raised if the same character is used in different roles."


def timeshift(self, shift='random'):
"""
Expand Down
5 changes: 4 additions & 1 deletion devicely/spacelabs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@

import pandas as pd

from ._compat import _to_csv_line_terminator


class SpacelabsReader:
"""
Expand Down Expand Up @@ -147,7 +149,8 @@ def write(self, path):
printing_df.replace('-9999', '""', inplace=True)
printing_df.replace('-9998', '"EB"', inplace=True)
printing_df.replace('-9997', '"AB"', inplace=True)
printing_df.to_csv(file, header=None, index=None, quoting=csv.QUOTE_NONE, line_terminator='\n')
printing_df.to_csv(file, header=None, index=None, quoting=csv.QUOTE_NONE,
**{_to_csv_line_terminator: '\n'})

xml_node = ET.Element('XML')
xml_node.extend(self._dict_to_etree(self.metadata))
Expand Down
6 changes: 5 additions & 1 deletion devicely/time_stamp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

import pandas as pd

from ._compat import _to_csv_line_terminator


class TimeStampReader:
"""
Read, timeshift and write data generated by the Android app TimeStamp
Expand Down Expand Up @@ -45,7 +48,8 @@ def write(self, path):

df_to_write = self.data.reset_index()[['tag_number', 'time', 'tag']]
df_to_write.time = df_to_write.time.dt.strftime("%Y/%-m/%-d(%a)\u3000%H:%M:%S").str.lower()
df_to_write.to_csv(path, header=None, index=None, line_terminator='\n')
df_to_write.to_csv(path, header=None, index=None,
**{_to_csv_line_terminator: '\n'})

def timeshift(self, shift='random'):
"""
Expand Down