From 8e80ecd3a0f6154af62637df7532f69e2a5b54d5 Mon Sep 17 00:00:00 2001
From: begoat <hylbegoat@gmail.com>
Date: Mon, 2 Aug 2021 13:11:59 +0800
Subject: [PATCH 1/5] feat(data.py): add custom datareader
---
pandas_datareader/__init__.py | 4 +++
pandas_datareader/data.py | 68 +++++++++++++++++++++++++++++++++--
2 files changed, 70 insertions(+), 2 deletions(-)
diff --git a/pandas_datareader/__init__.py b/pandas_datareader/__init__.py
index a792a806..616a75b9 100644
--- a/pandas_datareader/__init__.py
+++ b/pandas_datareader/__init__.py
@@ -28,6 +28,8 @@
get_records_iex,
get_summary_iex,
get_tops_iex,
+ get_custom_datareader,
+ register_custom_datareader,
)
PKG = os.path.dirname(__file__)
@@ -62,6 +64,8 @@
"get_data_tiingo",
"get_iex_data_tiingo",
"get_data_alphavantage",
+ "get_custom_datareader",
+ "register_custom_datareader",
"test",
]
diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py
index c2d6223a..4e4a1d8a 100644
--- a/pandas_datareader/data.py
+++ b/pandas_datareader/data.py
@@ -7,7 +7,7 @@
import warnings
from pandas.util._decorators import deprecate_kwarg
-
+from pandas_datareader.base import _BaseReader
from pandas_datareader.av.forex import AVForexReader
from pandas_datareader.av.quotes import AVQuotesReader
from pandas_datareader.av.sector import AVSectorPerformanceReader
@@ -60,9 +60,11 @@
"get_iex_book",
"get_dailysummary_iex",
"get_data_stooq",
+ "register_custom_datareader",
"DataReader",
]
+custom_datareader = {}
def get_data_alphavantage(*args, **kwargs):
return AVTimeSeriesReader(*args, **kwargs).read()
@@ -270,6 +272,55 @@ def get_iex_book(*args, **kwargs):
return IEXDeep(*args, **kwargs).read()
+def register_custom_datareader(custom_name, custom_class):
+ """
+ Registers a custom datareader to be used
+
+ Parameters
+ ----------
+ custom_name : str
+ A string represents name you want to give to your custom class
+ custom_class : _BaseReader
+ A class that extends _BaseReader
+
+ Returns
+ -------
+ True if successful, Error otherwise.
+ """
+ custom_datareader[custom_name] = custom_class
+ return True
+
+def unregister_custom_datareader(custom_name):
+ """
+ Unregisters a custom datareader to be used
+
+ Parameters
+ ----------
+ custom_name : str
+ A string represents name you want to give to your custom class
+
+ Returns
+ -------
+ True if successful, Error otherwise.
+ """
+ del custom_datareader[custom_name]
+ return True
+
+def get_custom_datareader(custom_name):
+ """
+ Get a custom datareader registered before
+
+ Parameters
+ ----------
+ custom_name : str
+ A string represents name you gave to your custom class
+
+ Returns
+ -------
+ Class registered before
+ """
+ return custom_datareader[custom_name]
+
@deprecate_kwarg("access_key", "api_key")
def DataReader(
name,
@@ -329,6 +380,7 @@ def DataReader(
ff = DataReader("6_Portfolios_2x3", "famafrench")
ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
"""
+ custom_source = list(custom_datareader.keys())
expected_source = [
"yahoo",
"iex",
@@ -360,7 +412,7 @@ def DataReader(
"av-intraday",
"econdb",
"naver",
- ]
+ ] + custom_source
if data_source not in expected_source:
msg = "data_source=%r is not implemented" % data_source
@@ -668,6 +720,18 @@ def DataReader(
session=session,
).read()
+ elif data_source in custom_source:
+ CustomDataReader = get_custom_datareader(data_source)
+ return CustomDataReader(
+ symbols=name,
+ start=start,
+ end=end,
+ retry_count=retry_count,
+ pause=pause,
+ session=session,
+ api_key=api_key,
+ ).read()
+
else:
msg = "data_source=%r is not implemented" % data_source
raise NotImplementedError(msg)
From 8aeac36384de717a5d79f8fd5ff3286e60135639 Mon Sep 17 00:00:00 2001
From: begoat <hylbegoat@gmail.com>
Date: Mon, 2 Aug 2021 14:54:35 +0800
Subject: [PATCH 2/5] refactor(data.py): code style check
---
pandas_datareader/data.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py
index 4e4a1d8a..4dc78348 100644
--- a/pandas_datareader/data.py
+++ b/pandas_datareader/data.py
@@ -66,6 +66,7 @@
custom_datareader = {}
+
def get_data_alphavantage(*args, **kwargs):
return AVTimeSeriesReader(*args, **kwargs).read()
@@ -290,6 +291,7 @@ def register_custom_datareader(custom_name, custom_class):
custom_datareader[custom_name] = custom_class
return True
+
def unregister_custom_datareader(custom_name):
"""
Unregisters a custom datareader to be used
@@ -306,6 +308,7 @@ def unregister_custom_datareader(custom_name):
del custom_datareader[custom_name]
return True
+
def get_custom_datareader(custom_name):
"""
Get a custom datareader registered before
@@ -321,6 +324,7 @@ def get_custom_datareader(custom_name):
"""
return custom_datareader[custom_name]
+
@deprecate_kwarg("access_key", "api_key")
def DataReader(
name,
From bb67ab10b724f2d4fb5f17616d020c13ef875726 Mon Sep 17 00:00:00 2001
From: begoat <hylbegoat@gmail.com>
Date: Mon, 2 Aug 2021 14:55:14 +0800
Subject: [PATCH 3/5] test(test_data.py): add testcase for custom datasource
---
pandas_datareader/tests/test_data.py | 43 +++++++++++++++++++++++++++-
1 file changed, 42 insertions(+), 1 deletion(-)
diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py
index 4cdea1d0..6349c42e 100644
--- a/pandas_datareader/tests/test_data.py
+++ b/pandas_datareader/tests/test_data.py
@@ -1,7 +1,8 @@
from pandas import DataFrame
import pytest
-from pandas_datareader.data import DataReader
+from pandas_datareader.base import _DailyBaseReader
+from pandas_datareader.data import DataReader, register_custom_datareader
pytestmark = pytest.mark.stable
@@ -18,3 +19,43 @@ def test_read_fred(self):
def test_not_implemented(self):
with pytest.raises(NotImplementedError):
DataReader("NA", "NA")
+
+ def test_custom_reader_acc(self):
+ class DemoReader(_DailyBaseReader):
+ def __init__(
+ self,
+ symbols=None,
+ start=None,
+ end=None,
+ retry_count=3,
+ pause=0.1,
+ session=None,
+ api_key=None,
+ ):
+ super().__init__(
+ symbols=symbols,
+ start=start,
+ end=end,
+ retry_count=retry_count,
+ pause=pause,
+ session=session,
+ )
+
+ @property
+ def url(self):
+ return 'https://stooq.com/q/d/l/'
+
+ def _get_params(self, symbol):
+ params = {
+ 's': symbol,
+ 'i': "d"
+ }
+ return params
+
+ register_custom_datareader('demo', DemoReader)
+ result = DataReader('USDJPY', 'demo')
+ assert isinstance(result, DataFrame)
+
+ def test_custom_reader_fail(self):
+ with pytest.raises(NotImplementedError):
+ DataReader('USDJPY', 'demo1')
\ No newline at end of file
From d0f6a5a42ab8223cbbf29b9152eceaf7f1e10a03 Mon Sep 17 00:00:00 2001
From: William Huang <hylbegoat@gmail.com>
Date: Mon, 2 Aug 2021 20:34:16 +0800
Subject: [PATCH 4/5] refactor(test_data.py): add blank line at eof
---
pandas_datareader/tests/test_data.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py
index 6349c42e..db1419cc 100644
--- a/pandas_datareader/tests/test_data.py
+++ b/pandas_datareader/tests/test_data.py
@@ -58,4 +58,4 @@ def _get_params(self, symbol):
def test_custom_reader_fail(self):
with pytest.raises(NotImplementedError):
- DataReader('USDJPY', 'demo1')
\ No newline at end of file
+ DataReader('USDJPY', 'demo1')
From a1e7a8e435a4f71d93d3cf1bdc052e97c61afa28 Mon Sep 17 00:00:00 2001
From: William Huang <hylbegoat@gmail.com>
Date: Mon, 2 Aug 2021 20:35:48 +0800
Subject: [PATCH 5/5] refactor(test_data.py): black check test_file
---
pandas_datareader/tests/test_data.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/pandas_datareader/tests/test_data.py b/pandas_datareader/tests/test_data.py
index db1419cc..0a77a5e3 100644
--- a/pandas_datareader/tests/test_data.py
+++ b/pandas_datareader/tests/test_data.py
@@ -43,19 +43,16 @@ def __init__(
@property
def url(self):
- return 'https://stooq.com/q/d/l/'
+ return "https://stooq.com/q/d/l/"
def _get_params(self, symbol):
- params = {
- 's': symbol,
- 'i': "d"
- }
+ params = {"s": symbol, "i": "d"}
return params
- register_custom_datareader('demo', DemoReader)
- result = DataReader('USDJPY', 'demo')
+ register_custom_datareader("demo", DemoReader)
+ result = DataReader("USDJPY", "demo")
assert isinstance(result, DataFrame)
def test_custom_reader_fail(self):
with pytest.raises(NotImplementedError):
- DataReader('USDJPY', 'demo1')
+ DataReader("USDJPY", "demo1")