diff --git a/README.md b/README.md index 5c6829e455..515b0f9ac2 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,6 @@ $ ansible-galaxy collection install ./theforeman-foreman-*.tar.gz These dependencies are required for the Ansible controller, not the Foreman server. * [`PyYAML`](https://pypi.org/project/PyYAML/) -* [`requests`](https://pypi.org/project/requests/) * `rpm` for the RPM support in the `content_upload` module * `debian` for the DEB support in the `content_upload` module diff --git a/bindep.txt b/bindep.txt index 7bc5d2dca1..a111bf7f42 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,2 +1 @@ python3-rpm [(platform:redhat platform:base-py3)] -python38-requests [platform:centos-8 platform:rhel-8] diff --git a/plugins/callback/foreman.py b/plugins/callback/foreman.py index 2af963238d..de9e9a30ac 100644 --- a/plugins/callback/foreman.py +++ b/plugins/callback/foreman.py @@ -17,7 +17,6 @@ - This callback will report facts and task events to Foreman requirements: - whitelisting in configuration - - requests (python library) options: report_type: description: @@ -112,10 +111,9 @@ import time try: - import requests - HAS_REQUESTS = True + from ansible_collections.theforeman.foreman.plugins.module_utils.ansible_requests import RequestSession except ImportError: - HAS_REQUESTS = False + from plugins.module_utils.ansible_requests import RequestSession from ansible.module_utils._text import to_text from ansible.module_utils.common.json import AnsibleJSONEncoder @@ -208,10 +206,7 @@ def set_options(self, task_keys=None, var_options=None, direct=None): ssl_key = self.get_option('client_key') self.dir_store = self.get_option('dir_store') - if not HAS_REQUESTS: - self._disable_plugin(u'The `requests` python module is not installed') - - self.session = requests.Session() + self.session = RequestSession() if self.foreman_url.startswith('https://'): if not os.path.exists(ssl_cert): self._disable_plugin(u'FOREMAN_SSL_CERT %s not found.' % ssl_cert) @@ -236,7 +231,6 @@ def _ssl_verify(self, option): verify = option if verify is False: # is only set to bool if try block succeeds - requests.packages.urllib3.disable_warnings() self._display.warning( u"SSL verification of %s disabled" % self.foreman_url, ) @@ -265,7 +259,7 @@ def _send_data(self, data_type, report_type, host, data): headers = {'content-type': 'application/json'} response = self.session.post(url=url, data=json_data, headers=headers) response.raise_for_status() - except requests.exceptions.RequestException as err: + except Exception as err: self._display.warning(u'Sending data to Foreman at {url} failed for {host}: {err}'.format( host=to_text(host), err=to_text(err), url=to_text(self.foreman_url))) diff --git a/plugins/doc_fragments/foreman.py b/plugins/doc_fragments/foreman.py index 0069cadeca..30cab4f436 100644 --- a/plugins/doc_fragments/foreman.py +++ b/plugins/doc_fragments/foreman.py @@ -22,8 +22,6 @@ class ModuleDocFragment(object): # Foreman documentation fragment DOCUMENTATION = ''' -requirements: - - requests options: server_url: description: diff --git a/plugins/inventory/foreman.py b/plugins/inventory/foreman.py index 74f0f4a8fe..8114e0e767 100644 --- a/plugins/inventory/foreman.py +++ b/plugins/inventory/foreman.py @@ -12,8 +12,6 @@ DOCUMENTATION = ''' name: foreman short_description: Foreman inventory source - requirements: - - requests >= 1.1 description: - Get inventory hosts from Foreman. - Can use the Reports API (default) or the Hosts API to fetch information about the hosts. @@ -191,19 +189,14 @@ from ansible_collections.theforeman.foreman.plugins.module_utils._version import LooseVersion from time import sleep from ansible.errors import AnsibleError -from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils._text import to_native, to_text from ansible.module_utils.common._collections_compat import MutableMapping from ansible.plugins.inventory import BaseInventoryPlugin, Cacheable, to_safe_group_name, Constructable -# 3rd party imports try: - import requests - if LooseVersion(requests.__version__) < LooseVersion('1.1.0'): - raise ImportError - from requests.auth import HTTPBasicAuth - HAS_REQUESTS = True + from ansible_collections.theforeman.foreman.plugins.module_utils.ansible_requests import RequestSession except ImportError: - HAS_REQUESTS = False + from plugins.module_utils.ansible_requests import RequestSession class InventoryModule(BaseInventoryPlugin, Cacheable, Constructable): @@ -222,9 +215,6 @@ def __init__(self): self.cache_key = None self.use_cache = None - if not HAS_REQUESTS: - raise AnsibleError('This script requires python-requests 1.1 as a minimum version') - def verify_file(self, path): valid = False @@ -237,8 +227,8 @@ def verify_file(self, path): def _get_session(self): if not self.session: - self.session = requests.session() - self.session.auth = HTTPBasicAuth(self.get_option('user'), to_bytes(self.get_option('password'))) + self.session = RequestSession() + self.session.auth = (self.get_option('user'), self.get_option('password')) self.session.verify = self.get_option('validate_certs') return self.session diff --git a/plugins/module_utils/_apypie.py b/plugins/module_utils/_apypie.py index 034e5680d3..024afb1a52 100644 --- a/plugins/module_utils/_apypie.py +++ b/plugins/module_utils/_apypie.py @@ -243,18 +243,8 @@ def _prepare_route_params(self, input_dict): import os from urllib.parse import urljoin # type: ignore -try: - import requests -except ImportError: - pass - -try: - from requests_gssapi import HTTPKerberosAuth # type: ignore -except ImportError: - try: - from requests_kerberos import HTTPKerberosAuth # type: ignore - except ImportError: - HTTPKerberosAuth = None + +HTTPKerberosAuth = None NO_CONTENT = 204 @@ -307,7 +297,7 @@ def __init__(self, **kwargs): self.apidoc_cache_dir = kwargs.get('apidoc_cache_dir', apidoc_cache_dir_default) self.apidoc_cache_name = kwargs.get('apidoc_cache_name', self._find_cache_name()) - self._session = kwargs.get('session') or requests.Session() + self._session = kwargs.get('session') self._session.verify = kwargs.get('verify_ssl', True) self._session.headers['Accept'] = 'application/json;version={}'.format(self.api_version) diff --git a/plugins/module_utils/ansible_requests.py b/plugins/module_utils/ansible_requests.py new file mode 100644 index 0000000000..7259818e8d --- /dev/null +++ b/plugins/module_utils/ansible_requests.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# pylint: disable=raise-missing-from +# pylint: disable=super-with-arguments + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +import json +from ansible.module_utils._text import to_native +from ansible.module_utils import six +from ansible.module_utils.urls import Request +try: + from ansible.module_utils.urls import prepare_multipart +except ImportError: + def prepare_multipart(fields): + raise NotImplementedError + + +class RequestException(Exception): + def __init__(self, msg, response): + super(RequestException, self).__init__(msg) + self.response = response + + +class RequestResponse(object): + def __init__(self, resp): + self._resp = resp + self._body = None + + @property + def status_code(self): + if hasattr(self._resp, 'status'): + status = self._resp.status + elif hasattr(self._resp, 'code'): + status = self._resp.code + else: + status = self._resp.getcode() + return status + + @property + def headers(self): + return self._resp.headers + + @property + def url(self): + if hasattr(self._resp, 'url'): + url = self._resp.url + elif hasattr(self._resp, 'geturl'): + url = self._resp.geturl() + else: + url = "" + return url + + @property + def reason(self): + if hasattr(self._resp, 'reason'): + reason = self._resp.reason + else: + reason = "" + return reason + + @property + def body(self): + if self._body is None: + try: + self._body = self._resp.read() + except Exception: + pass + return self._body + + @property + def text(self): + return to_native(self.body) + + def raise_for_status(self): + http_error_msg = "" + + if 400 <= self.status_code < 500: + http_error_msg = "{0} Client Error: {1} for url: {2}".format(self.status_code, self.reason, self.url) + elif 500 <= self.status_code < 600: + http_error_msg = "{0} Server Error: {1} for url: {2}".format(self.status_code, self.reason, self.url) + + if http_error_msg: + raise RequestException(http_error_msg, response=self) + + def json(self, **kwargs): + return json.loads(to_native(self.body), **kwargs) + + +class RequestSession(Request): + def __init__(self, **kwargs): + self.use_gssapi = kwargs.pop('use_gssapi', False) + super().__init__(**kwargs) + + @property + def auth(self): + return (self.url_username, self.url_password) + + @auth.setter + def auth(self, value): + self.url_username, self.url_password = value + self.force_basic_auth = True + + @property + def verify(self): + return self.validate_certs + + @verify.setter + def verify(self, value): + self.validate_certs = value + + @property + def cert(self): + return (self.client_cert, self.client_key) + + @cert.setter + def cert(self, value): + self.client_cert, self.client_key = value + + def request(self, method, url, **kwargs): + validate_certs = kwargs.pop('verify', None) + params = kwargs.pop('params', None) + if params: + url += '?' + six.moves.urllib.parse.urlencode(params) + headers = kwargs.pop('headers', None) + data = kwargs.pop('data', None) + if data: + if not isinstance(data, six.string_types): + data = six.moves.urllib.parse.urlencode(data, doseq=True) + files = kwargs.pop('files', None) + if files: + ansible_files = {k: {'filename': v[0], 'content': v[1].read(), 'mime_type': v[2]} for (k, v) in files.items()} + _content_type, data = prepare_multipart(ansible_files) + if 'json' in kwargs: + # it can be `json=None`… + data = json.dumps(kwargs.pop('json')) + if headers is None: + headers = {} + headers['Content-Type'] = 'application/json' + try: + result = self.open(method, url, validate_certs=validate_certs, use_gssapi=self.use_gssapi, data=data, headers=headers, **kwargs) + return RequestResponse(result) + except six.moves.urllib.error.HTTPError as e: + return RequestResponse(e) + + def get(self, url, **kwargs): + return self.request('GET', url, **kwargs) + + def post(self, url, data=None, json=None, **kwargs): + return self.request('POST', url, data=data, json=json, **kwargs) diff --git a/plugins/module_utils/foreman_helper.py b/plugins/module_utils/foreman_helper.py index 300845438f..8e496b889d 100644 --- a/plugins/module_utils/foreman_helper.py +++ b/plugins/module_utils/foreman_helper.py @@ -21,7 +21,7 @@ from functools import wraps from ansible.module_utils.basic import AnsibleModule, missing_required_lib, env_fallback -from ansible.module_utils._text import to_bytes, to_native +from ansible.module_utils._text import to_native from ansible.module_utils import six try: @@ -32,9 +32,10 @@ try: try: from ansible_collections.theforeman.foreman.plugins.module_utils import _apypie as apypie + from ansible_collections.theforeman.foreman.plugins.module_utils.ansible_requests import RequestSession except ImportError: from plugins.module_utils import _apypie as apypie - import requests.exceptions + from plugins.module_utils.ansible_requests import RequestSession HAS_APYPIE = True APYPIE_IMP_ERR = None inflector = apypie.Inflector() @@ -615,12 +616,14 @@ def connect(self): verify_ssl = self._foremanapi_ca_path if (self._foremanapi_validate_certs and self._foremanapi_ca_path) else self._foremanapi_validate_certs self.foremanapi = apypie.ForemanApi( uri=self._foremanapi_server_url, - username=to_bytes(self._foremanapi_username), - password=to_bytes(self._foremanapi_password), + username=self._foremanapi_username, + password=self._foremanapi_password, verify_ssl=verify_ssl, - kerberos=self._foremanapi_use_gssapi, task_timeout=self.task_timeout, + session=RequestSession(use_gssapi=self._foremanapi_use_gssapi), ) + if self._foremanapi_use_gssapi: + self.foremanapi.call('users', 'extlogin') _status = self.status() self.foreman_version = LooseVersion(_status.get('version', '0.0.0')) @@ -1184,7 +1187,7 @@ def wait_for_task(self, task, ignore_errors=False): def fail_from_exception(self, exc, msg): fail = {'msg': msg} - if isinstance(exc, requests.exceptions.HTTPError): + if hasattr(exc, 'response'): try: response = exc.response.json() if 'error' in response: diff --git a/requirements.txt b/requirements.txt index 643fe99774..5500f007d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -requests>=2.4.2 PyYAML diff --git a/tests/vcr_python_wrapper.py b/tests/vcr_python_wrapper.py index 136fc1923e..c8d44d79c8 100755 --- a/tests/vcr_python_wrapper.py +++ b/tests/vcr_python_wrapper.py @@ -42,7 +42,7 @@ def body_json_l2_matcher(r1, r2): for i, v in enumerate(body1): assert body1[i] == body2[i], "body contents at position {} dont't match: '{}' vs '{}'".format(i, body1[i], body2[i]) # pylint:disable=all else: - assert r1.body == r2.body, "{} != {}".format(r1.body, r2.body) + assert r1.body == r2.body, "{} != {} ({}, {})".format(r1.body, r2.body, r1.headers, r2.headers) def _query_without_search_matcher(r1, r2, path): diff --git a/vendor.py b/vendor.py index 29518545f1..a0551b4109 100644 --- a/vendor.py +++ b/vendor.py @@ -29,6 +29,9 @@ # empty lines trigger buffer flushes if line == '': + # we got an empty try/except because we dropped all code inbetween + if len(buffer_lines) >= 2 and buffer_lines[0] == 'try:' and buffer_lines[1].startswith('except'): + buffer_lines.clear() output_lines.extend(buffer_lines) buffer_lines.clear() if output_lines and output_lines[-1] != '': @@ -36,9 +39,12 @@ # drop apypie imports (we have one file now) and future imports (they are already present in the header) elif line.startswith('from apypie') or line.startswith('from __future__'): continue - # we can't just import requests, Ansible's "import" sanity test fails without the try/except - elif line == 'import requests': - output_lines.extend(['try:', ' import requests', 'except ImportError:', ' pass']) + # drop requests imports, we use a different implementation + elif line in ['import requests', ' from requests_gssapi import HTTPKerberosAuth # type: ignore', + ' from requests_kerberos import HTTPKerberosAuth # type: ignore']: + continue + elif line == ' HTTPKerberosAuth = None': + output_lines.append(line.strip()) # drop blocks that only handle typing imports (fenced by either try or if TYPE_CHECKING) elif line in ['try:', 'if TYPE_CHECKING:'] or buffer_lines: buffer_lines.append(line) @@ -54,6 +60,8 @@ # inject a blank line before class or import statements if (line.startswith('class ') or line.startswith('import ') or line.startswith('def ')) and not output_lines[-1].startswith('import '): output_lines.append('') + if line.endswith(' or requests.Session()'): + line = line.replace(' or requests.Session()', '') output_lines.append(line) # anything left in the buffer? flush it!