From 64dd6dde943def8c55681b709f16784c5c7318ee Mon Sep 17 00:00:00 2001 From: mbirger Date: Tue, 4 Sep 2018 08:40:11 +0200 Subject: [PATCH 1/2] Timeout parameters passed to Elasticsearch object Create timeout and retry on timeout parameters in the handler constuructor. Parameters passed to Elasticsearch object, and used while performing bulk_send(). --- cmreslogging/handlers.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 52e250a..53baa8e 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -75,6 +75,8 @@ class IndexNameFrequency(Enum): __DEFAULT_ES_DOC_TYPE = 'python_log' __DEFAULT_RAISE_ON_EXCEPTION = False __DEFAULT_TIMESTAMP_FIELD_NAME = "timestamp" + __DEFAULT_ES_RETRY_ON_TIMEOUT = False + __DEFAULT_ES_TIMEOUT = 10 __LOGGING_FILTER_FIELDS = ['msecs', 'relativeCreated', @@ -138,7 +140,9 @@ def __init__(self, es_doc_type=__DEFAULT_ES_DOC_TYPE, es_additional_fields=__DEFAULT_ADDITIONAL_FIELDS, raise_on_indexing_exceptions=__DEFAULT_RAISE_ON_EXCEPTION, - default_timestamp_field_name=__DEFAULT_TIMESTAMP_FIELD_NAME): + default_timestamp_field_name=__DEFAULT_TIMESTAMP_FIELD_NAME, + es_retry_on_timeout=__DEFAULT_ES_RETRY_ON_TIMEOUT, + es_timeout=__DEFAULT_ES_TIMEOUT): """ Handler constructor :param hosts: The list of hosts that elasticsearch clients will connect. The list can be provided @@ -172,6 +176,10 @@ def __init__(self, to the logs, such the application, environment, etc. :param raise_on_indexing_exceptions: A boolean, True only for debugging purposes to raise exceptions caused when + :param es_retry_on_timeout: A bool value, passed to Elasticsearch object. Specifies if bulk_send retry + sending logs after timeout. + :param es_timeout: An integer value, in seconds, passed to Elasticsearch object. Specifies client-side + timeout when performing bulk_send. :return: A ready to be used CMRESHandler. """ logging.Handler.__init__(self) @@ -194,6 +202,8 @@ def __init__(self, 'host_ip': socket.gethostbyname(socket.gethostname())}) self.raise_on_indexing_exceptions = raise_on_indexing_exceptions self.default_timestamp_field_name = default_timestamp_field_name + self.es_retry_on_timeout = es_retry_on_timeout + self.es_timeout = es_timeout self._client = None self._buffer = [] @@ -215,7 +225,9 @@ def __get_es_client(self): use_ssl=self.use_ssl, verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, - serializer=self.serializer) + serializer=self.serializer, + retry_on_timeout=self.es_retry_on_timeout, + timeout=self.es_timeout) return self._client if self.auth_type == CMRESHandler.AuthType.BASIC_AUTH: @@ -225,7 +237,9 @@ def __get_es_client(self): use_ssl=self.use_ssl, verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, - serializer=self.serializer) + serializer=self.serializer, + retry_on_timeout=self.es_retry_on_timeout, + timeout=self.es_timeout) return self._client if self.auth_type == CMRESHandler.AuthType.KERBEROS_AUTH: @@ -237,7 +251,9 @@ def __get_es_client(self): verify_certs=self.verify_certs, connection_class=RequestsHttpConnection, http_auth=HTTPKerberosAuth(mutual_authentication=DISABLED), - serializer=self.serializer) + serializer=self.serializer, + retry_on_timeout=self.es_retry_on_timeout, + timeout=self.es_timeout) if self.auth_type == CMRESHandler.AuthType.AWS_SIGNED_AUTH: if not AWS4AUTH_SUPPORTED: @@ -250,8 +266,9 @@ def __get_es_client(self): use_ssl=self.use_ssl, verify_certs=True, connection_class=RequestsHttpConnection, - serializer=self.serializer - ) + serializer=self.serializer, + retry_on_timeout=self.es_retry_on_timeout, + timeout=self.es_timeout) return self._client raise ValueError("Authentication method not supported") From 1c92ac90d17b68d7317270d19abaaad0f7f10251 Mon Sep 17 00:00:00 2001 From: Konstantin Schubert Date: Wed, 24 Apr 2019 22:36:00 +0200 Subject: [PATCH 2/2] Apply docstring suggestions from code review #54 Co-Authored-By: kusha --- cmreslogging/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 53baa8e..5a84a8a 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -176,9 +176,9 @@ def __init__(self, to the logs, such the application, environment, etc. :param raise_on_indexing_exceptions: A boolean, True only for debugging purposes to raise exceptions caused when - :param es_retry_on_timeout: A bool value, passed to Elasticsearch object. Specifies if bulk_send retry + :param es_retry_on_timeout: A boolean that defines if sending of logs should be re-tried after a timeout. sending logs after timeout. - :param es_timeout: An integer value, in seconds, passed to Elasticsearch object. Specifies client-side + :param es_timeout: An int that specifies, in seconds, the timeout on the client side when sending logs. timeout when performing bulk_send. :return: A ready to be used CMRESHandler. """