Merge pull request #366 from Chaffelson/retirePy2Automation

ottobackwards · web-flow · commit f17ed6007470 · 2024-10-07T07:46:29.000-04:00
Prepare v0.21.0 with py2 semi-retirement
diff --git a/.gitignore b/.gitignore
@@ -157,3 +157,9 @@ fabric.properties
 
 # Stupid OSX exclusions
 /.vscode
+
+.DS_Store
+# Coverage reports
+coverage*
+.coverage
+htmlcov/
diff --git a/docs/devnotes.rst b/docs/devnotes.rst
@@ -48,10 +48,16 @@ Instructions::
     I recommend you install PyEnv to manage Python versions `sudo curl https://pyenv.run | bash`
     Follow the instructions to set up your .bashrc
     To build various versions of Python for testing you may also need `sudo dnf install bzip2-devel openssl-devel libffi-devel zlib-devel readline-devel sqlite-devel -y`
-    Install the latest supported version of Python for your main dev environment `pyenv install 3.9 2.7`
-    Set these versions as global in pyenv so tox can see them. Use the actual versions with the command `pyenv global 3.9.16 2.7.62`
+    Install the latest supported version of Python for your main dev environment `pyenv install 3.9 2.7 3.12`
+    Set these versions as global in pyenv so tox can see them. Use the actual versions with the command `pyenv global 3.9.16 2.7.18 3.12.2`
     You'll want to stand up the two sets of NiFi containers for testing. resources/docker/tox-full for default and regression tests, and resources/docker/secure for tests under auth.
     You can switch between the tests by changing flags in tests/conftest.py around line 17.
+    Python3 can be tested automatically using Tox.
+    Python2 can be tested using the following steps within a Python2 virtualenv:
+    1. Install requirements: pip install -r requirements.txt
+    2. Install dev requirements: `pip install -r requirements_dev.txt`
+    3. Install package in editable mode with test support: `pip install -e .[test]`
+    4. Run tests: `pytest -v -s tests --tb=long -W ignore::urllib3.exceptions.InsecureRequestWarning`
 
 Setup Code Signing
 ------------------
@@ -194,7 +200,6 @@ This assumes you have virtualenvwrapper, git, and appropriate python versions in
     # Run appropriate tests, such as usage tests etc.
     deactivate
     Push changes to Github
-    Check build on TravisCI
     Check dockerhub automated build
     # You may have to reactivate your original virtualenv
     twine upload dist/*
diff --git a/docs/history.rst b/docs/history.rst
@@ -2,6 +2,27 @@
 History
 =======
 
+0.21.0 (2024-07-14)
+-------------------
+
+| Updated client for NiFi & Registry 1.27.0 release
+
+* Fix API model generator by @michaelarnauts in https://github.com/Chaffelson/nipyapi/pull/356
+* issue-360: handle -9 error messages better by @ottobackwards in https://github.com/Chaffelson/nipyapi/pull/361
+* Handle plain text response types so json values are correctly returned by @michaelarnauts in https://github.com/Chaffelson/nipyapi/pull/358
+* update clients to 1.27.0 by @Chaffelson in https://github.com/Chaffelson/nipyapi/pull/365
+* Simplified the use of setattr in recurse_flow, flatten, and list_all_by_kind methods in nipyapi/canvas.py. 
+* Added support for key_password in the Configuration class and its usage in nipyapi/nifi/rest.py and nipyapi/registry/rest.py. 
+* Fixed the method to retrieve HTTP headers in nipyapi/nifi/rest.py and nipyapi/registry/rest.py. 
+* Fixed issue #326 where the latest flow version was not being deployed by default
+* Updated pylintrc to match more modern python standards
+* Fixes nipyapi.nifi.ProcessGroupsApi.upload_process_group_with_http_info() incomplete #310
+* VersionedReportingTask added with appropriate functions
+* Move docker requirement to extras to avoid dependency install during standard usage
+* Set latest python3 version to 3.12
+* Deprecate usage of py.test in favour of newer pytest.
+* Update readme
+
 0.20.0 (2024-04-14)
 -------------------
 
diff --git a/docs/nipyapi-docs/nipyapi.nifi.models.rst b/docs/nipyapi-docs/nipyapi.nifi.models.rst
@@ -188,38 +188,6 @@ nipyapi.nifi.models.batch\_size module
     :undoc-members:
     :show-inheritance:
 
-nipyapi.nifi.models.bucket module
----------------------------------
-
-.. automodule:: nipyapi.nifi.models.bucket
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nipyapi.nifi.models.bucket\_dto module
---------------------------------------
-
-.. automodule:: nipyapi.nifi.models.bucket_dto
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nipyapi.nifi.models.bucket\_entity module
------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.bucket_entity
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nipyapi.nifi.models.buckets\_entity module
-------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.buckets_entity
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nipyapi.nifi.models.bulletin\_board\_dto module
 -----------------------------------------------
 
@@ -1036,14 +1004,6 @@ nipyapi.nifi.models.peers\_entity module
     :undoc-members:
     :show-inheritance:
 
-nipyapi.nifi.models.permissions module
---------------------------------------
-
-.. automodule:: nipyapi.nifi.models.permissions
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nipyapi.nifi.models.permissions\_dto module
 -------------------------------------------
 
@@ -1388,30 +1348,6 @@ nipyapi.nifi.models.queue\_size\_dto module
     :undoc-members:
     :show-inheritance:
 
-nipyapi.nifi.models.registry\_client\_entity module
----------------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.registry_client_entity
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nipyapi.nifi.models.registry\_clients\_entity module
-----------------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.registry_clients_entity
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-nipyapi.nifi.models.registry\_dto module
-----------------------------------------
-
-.. automodule:: nipyapi.nifi.models.registry_dto
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nipyapi.nifi.models.relationship\_dto module
 --------------------------------------------
 
@@ -1916,14 +1852,6 @@ nipyapi.nifi.models.versioned\_controller\_service module
     :undoc-members:
     :show-inheritance:
 
-nipyapi.nifi.models.versioned\_flow module
-------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.versioned_flow
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 nipyapi.nifi.models.versioned\_flow\_coordinates module
 -------------------------------------------------------
 
@@ -1948,13 +1876,6 @@ nipyapi.nifi.models.versioned\_flow\_entity module
     :undoc-members:
     :show-inheritance:
 
-nipyapi.nifi.models.versioned\_flow\_snapshot module
-----------------------------------------------------
-
-.. automodule:: nipyapi.nifi.models.versioned_flow_snapshot
-    :members:
-    :undoc-members:
-    :show-inheritance:
 
 nipyapi.nifi.models.versioned\_flow\_snapshot\_entity module
 ------------------------------------------------------------
diff --git a/nipyapi/__init__.py b/nipyapi/__init__.py
@@ -9,7 +9,7 @@
 
 __author__ = """Daniel Chaffelson"""
 __email__ = 'chaffelson@gmail.com'
-__version__ = '0.19.1'
+__version__ = '0.21.0'
 __all__ = ['canvas', 'system', 'templates', 'config', 'nifi', 'registry',
            'versioning', 'demo', 'utils', 'security', 'parameters']
 
diff --git a/nipyapi/canvas.py b/nipyapi/canvas.py
@@ -179,7 +179,8 @@ def flatten(parent_pg):
             Generator for all ProcessGroupEntities, eventually
         """
         for child_pg in parent_pg.process_group_flow.flow.process_groups:
-            yield from flatten(child_pg.nipyapi_extended)
+            for item in flatten(child_pg.nipyapi_extended):
+                yield item
             yield child_pg
 
     # Recurse children
diff --git a/nipyapi/security.py b/nipyapi/security.py
@@ -721,7 +721,7 @@ def create_access_policy(resource, action, r_id=None, service="nifi"):
         )
 
 
-# pylint: disable=R0913,R0917
+# pylint: disable=R0913
 def set_service_ssl_context(
     service="nifi",
     ca_file=None,
diff --git a/nipyapi/utils.py b/nipyapi/utils.py
@@ -18,6 +18,7 @@
 from packaging import version
 import six
 from ruamel.yaml import YAML
+from ruamel.yaml.compat import StringIO
 import requests
 from requests.models import Response
 from future.utils import raise_from as _raise
@@ -57,23 +58,22 @@ def dump(obj, mode='json'):
     assert mode in ['json', 'yaml']
     api_client = nipyapi.nifi.ApiClient()
     prepared_obj = api_client.sanitize_for_serialization(obj)
-    try:
-        out = json.dumps(
-            obj=prepared_obj,
-            sort_keys=True,
-            indent=4
-        )
-    except TypeError as e:
-        raise e
     if mode == 'json':
-        return out
+        try:
+            return json.dumps(
+                obj=prepared_obj,
+                sort_keys=True,
+                indent=4
+            )
+        except TypeError as e:
+            raise e
     if mode == 'yaml':
         # Use 'safe' loading to prevent arbitrary code execution
         yaml = YAML(typ='safe', pure=True)
         # Create a StringIO object to act as the stream
-        stream = io.StringIO()
+        stream = StringIO()
         # Dump to the StringIO stream
-        yaml.dump(json.loads(out), stream)
+        yaml.dump(prepared_obj, stream)
         # Return the contents of the stream as a string
         return stream.getvalue()
     raise ValueError("Invalid dump Mode specified {0}".format(mode))
@@ -350,7 +350,7 @@ def set_endpoint(endpoint_url, ssl=False, login=False,
     return True
 
 
-# pylint: disable=R0913,R0902,R0917
+# pylint: disable=R0913,R0902
 class DockerContainer():
     """
     Helper class for Docker container automation without using Ansible
diff --git a/nipyapi/versioning.py b/nipyapi/versioning.py
@@ -217,7 +217,7 @@ def get_flow_in_bucket(bucket_id, identifier, identifier_type='name',
         obj, identifier, identifier_type, greedy=greedy)
 
 
-# pylint: disable=R0913,R0917
+# pylint: disable=R0913
 def save_flow_ver(process_group, registry_client, bucket, flow_name=None,
                   flow_id=None, comment='', desc='', refresh=True,
                   force=False):
@@ -696,7 +696,7 @@ def import_flow_version(bucket_id, encoded_flow=None, file_path=None,
     )
 
 
-# pylint: disable=R0913,R0917
+# pylint: disable=R0913
 def deploy_flow_version(parent_id, location, bucket_id, flow_id, reg_client_id,
                         version=None):
     """
diff --git a/pylintrc b/pylintrc
@@ -429,8 +429,6 @@ disable=raw-checker-failed,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        use-implicit-booleaness-not-comparison-to-string,
-        use-implicit-booleaness-not-comparison-to-zero,
         bare-except,
         invalid-name,
         C0103, E1101, R1710, E0401, C0209
@@ -471,11 +469,6 @@ max-nested-blocks=5
 # printed.
 never-returning-functions=sys.exit,argparse.parse_error
 
-# Let 'consider-using-join' be raised when the separator to join on would be
-# non-empty (resulting in expected fixes of the type: ``"- " + " -
-# ".join(items)``)
-suggest-join-with-non-empty-separator=yes
-
 
 [REPORTS]
 
diff --git a/requirements.txt b/requirements.txt
@@ -19,10 +19,8 @@ requests[security]>=2.18
 pysocks>=1.7.1
 
 # Import Export and Utils implementation
-ruamel.yaml<0.18
-
-# Demo deployment automation
-docker>=2.5.1; extra == "demo"
+ruamel.yaml>=0.16.3,<0.18; python_version < '3'
+ruamel.yaml>=0.16.3; python_version >= '3'
 
 # xml to json parsing
 xmltodict>=0.12.0
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -4,21 +4,21 @@ pip>=9.0.1
 
 # Project management and Deployment
 bumpversion>=0.5.3
-watchdog>=0.8.3,<1.0.0  # pyup: ignore
-twine>=1.9.1,<2.0.0  # pyup: ignore
+watchdog>=0.8.3
+twine>=1.9.1
 virtualenvwrapper>=4.8
 virtualenv>=16.0.0  # required for tox 3.14.2 but not forced
 
 # Testing
-tox>=2.9.1
+tox>=3.28.0
 flake8>=3.6.0
 coverage>=4.4.1
 coveralls>=1.2.0
-pytest>=7.2.0
+pytest  # handled in setup.py
 nose>=1.3.7
 pluggy>=0.3.1
 pylint>=1.7.4
-deepdiff>=3.3.0,<4.0  # pyup: ignore
+deepdiff>=3.3.0
 
 # Docs
 Sphinx>=1.6.3
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.20.0
+current_version = 0.21.0
 commit = True
 tag = True
 
@@ -13,9 +13,3 @@ replace = __version__ = '{new_version}'
 
 [bdist_wheel]
 universal = 1
-
-[flake8]
-exclude = docs
-
-[aliases]
-test = pytest
diff --git a/setup.py b/setup.py
@@ -11,11 +11,20 @@
 with open('docs/history.rst') as history_file:
     history = history_file.read()
 
-proj_version = '0.20.0'
+proj_version = '0.21.0'
 
 with open('requirements.txt') as reqs_file:
     requirements = reqs_file.read().splitlines()
 
+tests_require=[
+    'pytest>=7.2.0;python_version>="3"',
+    'pytest>=6,<4;python_version<"3"',
+]
+
+extras_require={
+    'demo': ['docker>=2.5.1']
+}
+
 setup(
     name='nipyapi',
     version=proj_version,
@@ -31,9 +40,8 @@
     ),
     include_package_data=True,
     install_requires=requirements,
-    extras_require={
-        'demo': ['docker>=2.5.1']
-    },
+    tests_require=tests_require,
+    extras_require=extras_require,
     license="Apache Software License 2.0",
     zip_safe=False,
     keywords=['nipyapi', 'nifi', 'api', 'wrapper'],
diff --git a/tox.ini b/tox.ini

Original file line number	Diff line number	Diff line change
`@@ -721,7 +721,7 @@ def create_access_policy(resource, action, r_id=None, service="nifi"):`
`721`	`721`	`)`
`722`	`722`
`723`	`723`
`724`		`-# pylint: disable=R0913,R0917`
	`724`	`+# pylint: disable=R0913`
`725`	`725`	`def set_service_ssl_context(`
`726`	`726`	`service="nifi",`
`727`	`727`	`ca_file=None,`