WIP - New style io (#43)

* 🎉 new style reader

* 🤝 update static meta data

* 💚 adapter the exception name change

* 🎉 new style writer

* 💚 add content-xlsx reader plugin

* 📚 update change log and bump version number

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 🔥 remove unwanted matrix

* 🎉 register in content reader

Co-authored-by: chfw <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,9 @@
 With your PR, here is a check list:
 
-- [ ] Has Test cases written
-- [ ] Has all code lines tested
+- [ ] Has test cases written?
+- [ ] Has all code lines tested?
 - [ ] Has `make format` been run?
 - [ ] Please update CHANGELOG.yml(not CHANGELOG.rst)
 - [ ] Passes all Travis CI builds
 - [ ] Has fair amount of documentation if your change is complex
-- [ ] run 'make format' so as to confirm the pyexcel organisation's coding style
-- [ ] Please add yourself to 'contributors' section of pyexcel-xlsx.yml (if not found, please use CONTRIBUTORS.rst)
 - [ ] Agree on NEW BSD License for your contribution

.gitignore

@@ -540,3 +540,7 @@ cscope.files
 cscope.out
 cscope.in.out
 cscope.po.out
+
+
+# remove moban hash dictionary
+.moban.hashes

.moban.d/custom_travis.yml.jj2

@@ -1,11 +1,5 @@
 {% extends "travis.yml.jj2" %}
 
-{%block extra_matrix %}
-matrix:
-  include:
-    - python: 3.6
-      env: MINREQ=1
-{%endblock%}
 {%block custom_python_versions%}
 python:
   - 3.8

.travis.yml

@@ -7,10 +7,6 @@ python:
   - 3.8
   - 3.7
   - 3.6
-matrix:
-  include:
-    - python: 3.6
-      env: MINREQ=1
 
 stages:
   - lint

CHANGELOG.rst

@@ -1,6 +1,13 @@
 Change log
 ================================================================================
 
+0.6.0 - tbd
+--------------------------------------------------------------------------------
+
+**Updated**
+
+#. New style xlsx plugins, promoted by pyexcel-io v0.6.0.
+
 0.5.8 - 28.12.2019
 --------------------------------------------------------------------------------
 

CONTRIBUTORS.rst

@@ -1,9 +1,10 @@
 
+
 3 contributors
 ================================================================================
 
 In alphabetical order:
 
-* `Benoit Pierre <https://api.github.com/users/benoit-pierre>`_
-* `John Vandenberg <https://api.github.com/users/jayvdb>`_
-* `Stephen J. Fuhry <https://api.github.com/users/fuhrysteve>`_
+* `Benoit Pierre <https://github.com/benoit-pierre>`_
+* `John Vandenberg <https://github.com/jayvdb>`_
+* `Stephen J. Fuhry <https://github.com/fuhrysteve>`_

Makefile

@@ -6,9 +6,6 @@ test: lint
 install_test:
 	pip install -r tests/requirements.txt
 
-git-diff-check:
-	git diff --exit-code
-
 lint:
 	bash lint.sh
 

changelog.yml

@@ -1,6 +1,12 @@
 name: pyexcel-xlsx
 organisation: pyexcel
 releases:
+- changes:
+  - action: Updated
+    details:
+    - 'New style xlsx plugins, promoted by pyexcel-io v0.6.0.'
+  date: tbd
+  version: 0.6.0
 - changes:
   - action: Updated
     details:

docs/source/conf.py

@@ -26,7 +26,7 @@
 copyright = '2015-2019 Onni Software Ltd.'
 author = 'chfw'
 # The short X.Y version
-version = '0.5.8'
+version = '0.6.0'
 # The full version, including alpha/beta/rc tags
 release = '0.5.8'
 

lint.sh

@@ -1,3 +1,2 @@
 pip install flake8
-flake8 . --exclude=.moban.d,docs,setup.py --builtins=unicode,xrange,long
-python setup.py checkdocs
+flake8 --exclude=.moban.d,docs,setup.py   --builtins=unicode,xrange,long . && python setup.py checkdocs

pyexcel-xlsx.yml

@@ -2,13 +2,16 @@ overrides: "pyexcel.yaml"
 name: "pyexcel-xlsx"
 project: "pyexcel-xlsx"
 nick_name: xlsx
-version: 0.5.8
-current_version: 0.5.8
+version: 0.6.0
+current_version: 0.6.0
 release: 0.5.8
 file_type: xlsx
 gitignore_language: Python
 is_on_conda: true
 dependencies:
   - openpyxl>=2.6.1
   - pyexcel-io>=0.5.3
+test_dependencies:
+  - pyexcel-xls
+  - pyexcel
 description: A wrapper library to read, manipulate and write data in xlsx and xlsm format

pyexcel_xlsx/__init__.py

@@ -10,15 +10,23 @@
 from pyexcel_io.io import get_data as read_data
 from pyexcel_io.io import isstream
 from pyexcel_io.io import save_data as write_data
-from pyexcel_io.plugins import IOPluginInfoChain
+from pyexcel_io.plugins import IOPluginInfoChainV2
 
 __FILE_TYPE__ = "xlsx"
-IOPluginInfoChain(__name__).add_a_reader(
+
+IOPluginInfoChainV2(__name__).add_a_reader(
     relative_plugin_class_path="xlsxr.XLSXBook",
+    locations=["file", "memory"],
+    file_types=[__FILE_TYPE__, "xlsm"],
+    stream_type="binary",
+).add_a_reader(
+    relative_plugin_class_path="xlsxr.XLSXBookInContent",
+    locations=["content"],
     file_types=[__FILE_TYPE__, "xlsm"],
     stream_type="binary",
 ).add_a_writer(
     relative_plugin_class_path="xlsxw.XLSXWriter",
+    locations=["file", "memory"],
     file_types=[__FILE_TYPE__, "xlsm"],
     stream_type="binary",
 )

pyexcel_xlsx/xlsxr.py

@@ -7,14 +7,14 @@
     :copyright: (c) 2015-2019 by Onni Software Ltd & its contributors
     :license: New BSD License
 """
-import openpyxl
+from io import BytesIO
 
-from pyexcel_io.book import BookReader
-from pyexcel_io.sheet import SheetReader
-from pyexcel_io._compact import OrderedDict, irange
+import openpyxl
+from pyexcel_io._compact import OrderedDict
+from pyexcel_io.plugin_api.abstract_sheet import ISheet
 
 
-class FastSheet(SheetReader):
+class FastSheet(ISheet):
     """
     Iterate through rows
     """
@@ -47,8 +47,8 @@ def __init__(self, cell_ranges):
         self.value = None
 
     def register_cells(self, registry):
-        for rowx in irange(self.__rl, self.__rh + 1):
-            for colx in irange(self.__cl, self.__ch + 1):
+        for rowx in range(self.__rl, self.__rh + 1):
+            for colx in range(self.__cl, self.__ch + 1):
                 key = "%s-%s" % (rowx, colx)
                 registry[key] = self
 
@@ -65,7 +65,8 @@ class SlowSheet(FastSheet):
     """
 
     def __init__(self, sheet, **keywords):
-        SheetReader.__init__(self, sheet, **keywords)
+        self._native_sheet = sheet
+        self._keywords = keywords
         self.__merged_cells = {}
         self.max_row = 0
         self.max_column = 0
@@ -124,73 +125,43 @@ def _merged_cells(self, row, column, value):
         return ret
 
 
-class XLSXBook(BookReader):
+class XLSXBook(object):
     """
     Open xlsx as read only mode
     """
 
-    def open(
+    def __init__(
         self,
-        file_name,
+        file_alike_object,
+        file_type,
         skip_hidden_sheets=True,
         detect_merged_cells=False,
         skip_hidden_row_and_column=True,
         **keywords
     ):
-
-        BookReader.open(self, file_name, **keywords)
         self.skip_hidden_sheets = skip_hidden_sheets
         self.skip_hidden_row_and_column = skip_hidden_row_and_column
         self.detect_merged_cells = detect_merged_cells
-        self._load_the_excel_file(file_name)
-
-    def open_stream(
-        self,
-        file_stream,
-        skip_hidden_sheets=True,
-        detect_merged_cells=False,
-        skip_hidden_row_and_column=True,
-        **keywords
-    ):
-        BookReader.open_stream(self, file_stream, **keywords)
-        self.skip_hidden_sheets = skip_hidden_sheets
-        self.skip_hidden_row_and_column = skip_hidden_row_and_column
-        self.detect_merged_cells = detect_merged_cells
-        self._load_the_excel_file(file_stream)
-
-    def read_sheet_by_name(self, sheet_name):
-        sheet = self._native_book[sheet_name]
-        if sheet is None:
-            raise ValueError("%s cannot be found" % sheet_name)
-        else:
-            return self.read_sheet(sheet)
+        self.keywords = keywords
+        self._load_the_excel_file(file_alike_object)
 
-    def read_sheet_by_index(self, sheet_index):
-        names = self._native_book.sheetnames
-        length = len(names)
-        if sheet_index < length:
-            return self.read_sheet_by_name(names[sheet_index])
+    def read_sheet(self, sheet_index):
+        native_sheet = self.content_array[sheet_index].sheet
+        if self.skip_hidden_row_and_column or self.detect_merged_cells:
+            sheet = SlowSheet(native_sheet, **self.keywords)
         else:
-            raise IndexError(
-                "Index %d of out bound %d" % (sheet_index, length)
-            )
+            sheet = FastSheet(native_sheet, **self.keywords)
+        return sheet
 
     def read_all(self):
         result = OrderedDict()
-        for sheet in self._native_book:
+        for index, sheet in enumerate(self._native_book):
             if self.skip_hidden_sheets and sheet.sheet_state == "hidden":
                 continue
-            data_dict = self.read_sheet(sheet)
-            result.update(data_dict)
+            sheet = self.read_sheet(index)
+            result.update({sheet.name: sheet.to_array()})
         return result
 
-    def read_sheet(self, native_sheet):
-        if self.skip_hidden_row_and_column or self.detect_merged_cells:
-            sheet = SlowSheet(native_sheet, **self._keywords)
-        else:
-            sheet = FastSheet(native_sheet, **self._keywords)
-        return {sheet.name: sheet.to_array()}
-
     def close(self):
         self._native_book.close()
         self._native_book = None
@@ -207,3 +178,26 @@ def _load_the_excel_file(self, file_alike_object):
             data_only=data_only_flag,
             read_only=read_only_flag,
         )
+        self.content_array = []
+        for sheet_name, sheet in zip(
+            self._native_book.sheetnames, self._native_book
+        ):
+            if self.skip_hidden_sheets and sheet.sheet_state == "hidden":
+                continue
+            self.content_array.append(NameObject(sheet_name, sheet))
+
+
+class XLSXBookInContent(XLSXBook):
+    """
+    Open xlsx as read only mode
+    """
+
+    def __init__(self, file_content, file_type, **keywords):
+        io = BytesIO(file_content)
+        super().__init__(io, file_type, **keywords)
+
+
+class NameObject(object):
+    def __init__(self, name, sheet):
+        self.name = name
+        self.sheet = sheet

pyexcel_xlsx/xlsxw.py

@@ -8,39 +8,41 @@
     :license: New BSD License
 """
 import openpyxl
+from pyexcel_io import constants
+from pyexcel_io.plugin_api.abstract_sheet import ISheetWriter
+from pyexcel_io.plugin_api.abstract_writer import IWriter
 
-from pyexcel_io.book import BookWriter
-from pyexcel_io.sheet import SheetWriter
 
-
-class XLSXSheetWriter(SheetWriter):
+class XLSXSheetWriter(ISheetWriter):
     """
     Write data into xlsx sheet
     """
 
-    def set_sheet_name(self, name):
-        self._native_sheet.title = name
-        self.current_row = 1
+    def __init__(self, xlsx_book, xlsx_sheet, sheet_name, **keywords):
+        if sheet_name is None:
+            sheet_name = constants.DEFAULT_SHEET_NAME
+        self._xlsx_book = xlsx_book
+        self._xlsx_sheet = xlsx_sheet
+        self._keywords = keywords
+        self._xlsx_sheet.title = sheet_name
 
     def write_row(self, array):
         """
         write a row into the file
         """
-        self._native_sheet.append(array)
+        self._xlsx_sheet.append(array)
+
+    def close(self):
+        pass
 
 
-class XLSXWriter(BookWriter):
+class XLSXWriter(IWriter):
     """
     Write data in write only mode
     """
 
-    def __init__(self):
-        BookWriter.__init__(self)
-        self.current_sheet = 0
-        self._native_book = None
-
-    def open(self, file_name, **keywords):
-        BookWriter.open(self, file_name, **keywords)
+    def __init__(self, file_alike_object, _, **keywords):
+        self._file_alike_object = file_alike_object
         self._native_book = openpyxl.Workbook(write_only=True)
 
     def create_sheet(self, name):
@@ -53,4 +55,3 @@ def close(self):
         This call actually save the file
         """
         self._native_book.save(filename=self._file_alike_object)
-        self._native_book = None

setup.py

@@ -29,7 +29,7 @@
 
 NAME = "pyexcel-xlsx"
 AUTHOR = "chfw"
-VERSION = "0.5.8"
+VERSION = "0.6.0"
 EMAIL = "[email protected]"
 LICENSE = "New BSD"
 DESCRIPTION = (

tests/requirements.txt

@@ -9,3 +9,5 @@ collective.checkdocs
 pygments
 moban
 moban_jinja2_github
+pyexcel-xls
+pyexcel