Merge pull request #1001 from bashtage/fixes-for-pandas-3

MAINT: Fix PDF for pandas 3 changes

Author: bashtage

ci/azure/azure_template_posix.yml

@@ -16,19 +16,22 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      python38_legacy:
-        python.version: '3.8'
-        PANDAS: 1.5.3
       python39_legacy:
         python.version: '3.9'
         PANDAS: 1.5.3
-      python310_recent:
-        python.version: '3.9'
+        NUMPY: 1.23.4
+      python310_legacy:
+        python.version: '3.10'
         PANDAS: 2.0.3
+        NUMPY: 1.24.4
       python311_latest:
         python.version: '3.11'
+        PANDAS: 2.1.4
+        NUMPY: 1.26.4
       python312_latest:
         python.version: '3.12'
+      python313_latest:
+        python.version: '3.13'
 
   steps:
   - task: UsePythonVersion@0
@@ -40,6 +43,9 @@ jobs:
       python -m pip install pip setuptools -U      
       python -m pip install -r requirements.txt
       python -m pip install -r requirements-dev.txt
+      if [[ -n ${NUMPY} ]]; then 
+        python -m pip install numpy==${NUMPY}
+      fi;
       if [[ -n ${PANDAS} ]]; then 
         python -m pip install pandas==${PANDAS}
       fi;
@@ -73,9 +79,8 @@ jobs:
       testRunTitle: 'Python $(python.version)'
     condition: succeededOrFailed()
 
-  - task: PublishCodeCoverageResults@1
+  - task: PublishCodeCoverageResults@2
     inputs:
-      codeCoverageTool: Cobertura
       summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
     condition: eq(variables['coverage'], 'true')
 

ci/azure/azure_template_windows.yml

@@ -7,7 +7,6 @@ parameters:
   # defaults for any parameters that are not specified
   name: ''
   vmImage: ''
-  coverage: true
 
 jobs:
 
@@ -16,19 +15,21 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      python38_legacy:
-        python.version: '3.8'
-        PANDAS: 1.5.3
       python39_legecy:
         python.version: '3.9'
         PANDAS: 1.5.3
+        NUMPY: 1.23.4
       python310_recent:
         python.version: '3.10'
-        PANDAS: 2.0.3
+        NUMPY: 1.24.4
       python311_latest:
         python.version: '3.11'
+        PANDAS: 2.1.4
+        NUMPY: 1.26.4
       python312_latest:
         python.version: '3.12'
+      python313_latest:
+        python.version: '3.13'
 
   steps:
   - task: UsePythonVersion@0
@@ -44,10 +45,13 @@ jobs:
     displayName: 'Install dependencies'
 
   - powershell: |
+      if ($null -ne $env:NUMPY) {
+        python -m pip install numpy==$env:NUMPY
+      }
       if ($null -ne $env:PANDAS) {
-        python -m pip install pandas==${PANDAS}
+        python -m pip install pandas==$env:PANDAS
       }
-    displayName: 'Update pandas (if needed)'
+    displayName: 'Update numpy and pandas (if needed)'
 
   - script: |
       python -m pip install -e . -vv
@@ -64,25 +68,12 @@ jobs:
 
   - script: |
       echo "Testing editable install"
-      if [[ ${COVERAGE} == "true" ]]; then
-        export COVERAGE_OPTS="--cov-config .coveragerc --cov=pandas_datareader --cov-report xml:coverage.xml --cov-report term"
-      fi
-      echo pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} pandas_datareader/tests
-      pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} pandas_datareader/tests
+      echo pytest --durations=25 pandas_datareader/tests
+      pytest --durations=25 pandas_datareader/tests
     displayName: 'Run tests'
 
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/test-results.xml'
       testRunTitle: 'Python $(python.version)'
     condition: succeededOrFailed()
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
-    condition: eq(variables['coverage'], 'true')
-
-  - bash: bash <(curl -s https://codecov.io/bash)
-    displayName: 'CodeCov upload'
-    condition: eq(variables['coverage'], 'true')

pandas_datareader/base.py

@@ -115,8 +115,9 @@ def _read_url_as_StringIO(self, url, params=None):
         if len(text) == 0:
             service = self.__class__.__name__
             raise OSError(
-                "{} request returned no data; check URL for invalid "
-                "inputs: {}".format(service, self.url)
+                "{} request returned no data; check URL for invalid inputs: {}".format(
+                    service, self.url
+                )
             )
         if isinstance(text, bytes):
             out.write(text.decode("utf-8"))

pandas_datareader/compat/__init__.py

@@ -1,4 +1,5 @@
 from io import StringIO
+import sys
 from urllib.error import HTTPError
 
 from pandas.api.types import is_list_like, is_number
@@ -12,7 +13,7 @@
     "assert_frame_equal",
     "is_list_like",
     "is_number",
-    "reduce",
+    "PYTHON_LT_3_10",
 ]
 
 
@@ -31,3 +32,6 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None, compression=None):
             filepath_or_buffer, encoding=encoding, compression=None
         )
         return tmp
+
+
+PYTHON_LT_3_10 = sys.version_info <= (3, 10)

pandas_datareader/data.py

@@ -683,8 +683,7 @@ def DataReader(
 def Options(symbol, data_source=None, session=None):
     if data_source is None:
         warnings.warn(
-            "Options(symbol) is deprecated, use Options(symbol,"
-            " data_source) instead",
+            "Options(symbol) is deprecated, use Options(symbol, data_source) instead",
             FutureWarning,
             stacklevel=2,
         )

pandas_datareader/famafrench.py

@@ -6,7 +6,7 @@
 from pandas import read_csv, to_datetime
 
 from pandas_datareader.base import _BaseReader
-from pandas_datareader.compat import StringIO
+from pandas_datareader.compat import PYTHON_LT_3_10, StringIO
 
 _URL = "http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/"
 _URL_PREFIX = "ftp/"
@@ -57,7 +57,7 @@ def _read_zipfile(self, url):
         with tempfile.TemporaryFile() as tmpf:
             tmpf.write(raw)
             with ZipFile(tmpf, "r") as zf:
-                try:                
+                try:
                     data = zf.open(zf.namelist()[0]).read().decode("utf-8", "ignore")
                 except UnicodeDecodeError:
                     data = zf.open(zf.namelist()[0]).read().decode(encoding="cp1252")
@@ -78,8 +78,6 @@ def read(self):
     def _read_one_data(self, url, params):
         params = {
             "index_col": 0,
-            "parse_dates": [0],
-            "date_parser": _parse_date_famafrench,
         }
 
         # headers in these files are not valid
@@ -89,7 +87,12 @@ def _read_one_data(self, url, params):
             else:
                 c = ["Count"]
             r = list(range(0, 105, 5))
-            params["names"] = ["Date"] + c + list(zip(r, r[1:], strict=False))
+
+            if PYTHON_LT_3_10:
+                additional_params = list(zip(r, r[1:]))  # noqa: B905
+            else:
+                additional_params = list(zip(r, r[1:], strict=False))
+            params["names"] = ["Date"] + c + additional_params
 
             if self.symbols != "Prior_2-12_Breakpoints":
                 params["skiprows"] = 1
@@ -111,12 +114,14 @@ def _read_one_data(self, url, params):
             start = 0 if not match else match.start()
 
             df = read_csv(StringIO("Date" + src[start:]), **params)
-            try:
-                idx_name = df.index.name  # hack for pandas 0.16.2
-                df = df.to_period(df.index.inferred_freq[:1])
-                df.index.name = idx_name
-            except Exception:
-                pass
+            if df.index.min() > 190000:
+                df.index = to_datetime(df.index.astype(str), format="%Y%m").to_period(
+                    freq="M"
+                )
+            else:
+                df.index = to_datetime(df.index.astype(str), format="%Y").to_period(
+                    freq="Y"
+                )
             df = df.truncate(self.start, self.end)
             datasets[i] = df
 

pandas_datareader/fred.py

@@ -58,8 +58,13 @@ def fetch_data(url, name):
                     ) from exc
                 raise
 
+        try:
+            data = [fetch_data(url, n) for url, n in zip(urls, names, strict=True)]
+        except TypeError:
+            # Python 3.9 only
+            data = [fetch_data(url, n) for url, n in zip(urls, names)]  # noqa: B905
         df = concat(
-            [fetch_data(url, n) for url, n in zip(urls, names, strict=True)],
+            data,
             axis=1,
             join="outer",
         )

pandas_datareader/iex/daily.py

@@ -15,7 +15,6 @@
 
 
 class IEXDailyReader(_DailyBaseReader):
-
     """
     Returns DataFrame of historical stock prices
     from symbols, over date range, start to end. To avoid being penalized by

pandas_datareader/moex.py

@@ -197,8 +197,9 @@ def read_all_boards(self):
 
         if len(dfs) == 0:
             raise OSError(
-                "{} returned no data; "
-                "check URL or correct a date".format(self.__class__.__name__)
+                "{} returned no data; check URL or correct a date".format(
+                    self.__class__.__name__
+                )
             )
         elif len(dfs) > 1:
             b = pd.concat(dfs, axis=0, join="outer", sort=True)
@@ -226,8 +227,9 @@ def _read_url_as_String(self, url, params=None):
         if len(text) == 0:
             service = self.__class__.__name__
             raise OSError(
-                "{} request returned no data; check URL for invalid "
-                "inputs: {}".format(service, self.url)
+                "{} request returned no data; check URL for invalid inputs: {}".format(
+                    service, self.url
+                )
             )
         if isinstance(text, bytes):
             text = text.decode("windows-1251")

pandas_datareader/stooq.py

@@ -2,7 +2,6 @@
 
 
 class StooqDailyReader(_DailyBaseReader):
-
     """
     Returns DataFrame/dict of Dataframes of historical stock prices from
     symbols, over date range, start to end.

pandas_datareader/tests/test_bankofcanada.py

@@ -41,10 +41,15 @@ def check_bankofcanada_inverted(self, code):
             date.today() - timedelta(days=30),
             date.today(),
         )
-
-        pairs = zip(
-            (1 / df)[symbol].tolist(), df_i[symbol_inverted].tolist(), strict=True
-        )
+        try:
+            pairs = zip(
+                (1 / df)[symbol].tolist(), df_i[symbol_inverted].tolist(), strict=True
+            )
+        except TypeError:
+            # Python 3.9 only
+            pairs = zip(  # noqa: B905
+                (1 / df)[symbol].tolist(), df_i[symbol_inverted].tolist()
+            )
         assert all(a - b < 0.01 for a, b in pairs)
 
     def test_bankofcanada_usd_count(self):

pandas_datareader/tests/test_famafrench.py

@@ -32,8 +32,10 @@ def test_get_available_datasets(self):
 
     def test_index(self):
         ff = web.DataReader("F-F_Research_Data_Factors", "famafrench")
-        assert ff[0].index.freq == "M"
-        assert ff[1].index.freq == "A-DEC"
+        # M is for legacy pandas < 2
+        assert ff[0].index.freq.name in ("ME", "M")
+        # A-DEC is for legacy pandas < 2
+        assert ff[1].index.freq.name in ("YE-DEC", "A-DEC")
 
     def test_f_f_research(self):
         results = web.DataReader(
@@ -49,49 +51,49 @@ def test_f_f_research(self):
             {
                 "Mkt-RF": [
                     -3.36,
-                    3.40,
+                    3.4,
                     6.31,
-                    2.00,
+                    2.0,
                     -7.89,
                     -5.57,
                     6.93,
                     -4.77,
                     9.54,
                     3.88,
-                    0.60,
+                    0.6,
                     6.82,
                 ],
                 "SMB": [
-                    0.40,
+                    0.4,
                     1.19,
                     1.48,
                     4.87,
                     0.09,
-                    -1.82,
-                    0.20,
-                    -3.00,
+                    -1.81,
+                    0.2,
+                    -3.0,
                     3.96,
-                    1.14,
-                    3.77,
+                    1.13,
+                    3.76,
                     0.73,
                 ],
                 "HML": [
                     0.43,
-                    3.23,
+                    3.22,
                     2.21,
                     2.89,
                     -2.44,
-                    -4.70,
+                    -4.7,
                     -0.31,
-                    -1.90,
+                    -1.9,
                     -3.16,
-                    -2.43,
+                    -2.42,
                     -0.96,
-                    3.70,
+                    3.69,
                 ],
                 "RF": [
-                    0.00,
-                    0.00,
+                    0.0,
+                    0.0,
                     0.01,
                     0.01,
                     0.01,
@@ -114,7 +116,7 @@ def test_f_f_research(self):
 
     def test_me_breakpoints(self):
         results = web.DataReader(
-            "ME_Breakpoints", "famafrench", start="2010-01-01", end="2010-12-01"
+            "ME_Breakpoints", "famafrench", start="2010-01-01", end="2010-12-31"
         )
         assert isinstance(results, dict)
         assert len(results) == 2