From 7de3d8ec9d39bf27601b5da414f3011c6f7f7424 Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Wed, 24 Mar 2021 12:47:44 +0200 Subject: [PATCH 01/11] added max retries --- cmreslogging/handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 52e250a..722b512 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -300,7 +300,8 @@ def flush(self): eshelpers.bulk( client=self.__get_es_client(), actions=actions, - stats_only=True + stats_only=True, + max_retries=10, ) except Exception as exception: if self.raise_on_indexing_exceptions: From fba2aaa1dc7fae14ef8f052c3ea837683262cd42 Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Wed, 24 Mar 2021 14:37:24 +0200 Subject: [PATCH 02/11] max retries --- cmreslogging/handlers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 722b512..a472d81 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -11,12 +11,14 @@ try: from requests_kerberos import HTTPKerberosAuth, DISABLED + CMR_KERBEROS_SUPPORTED = True except ImportError: CMR_KERBEROS_SUPPORTED = False try: from requests_aws4auth import AWS4Auth + AWS4AUTH_SUPPORTED = True except ImportError: AWS4AUTH_SUPPORTED = False @@ -133,6 +135,7 @@ def __init__(self, verify_ssl=__DEFAULT_VERIFY_SSL, buffer_size=__DEFAULT_BUFFER_SIZE, flush_frequency_in_sec=__DEFAULT_FLUSH_FREQ_INSEC, + max_retries=10, es_index_name=__DEFAULT_ES_INDEX_NAME, index_name_frequency=__DEFAULT_INDEX_FREQUENCY, es_doc_type=__DEFAULT_ES_DOC_TYPE, @@ -186,6 +189,7 @@ def __init__(self, self.verify_certs = verify_ssl self.buffer_size = buffer_size self.flush_frequency_in_sec = flush_frequency_in_sec + self.max_retries = max_retries self.es_index_name = es_index_name self.index_name_frequency = index_name_frequency self.es_doc_type = es_doc_type @@ -301,7 +305,7 @@ def flush(self): client=self.__get_es_client(), actions=actions, stats_only=True, - max_retries=10, + max_retries=self.max_retries, ) except Exception as exception: if self.raise_on_indexing_exceptions: From 29e8c407bdc1a0ed9bf142b2a51adb5c89684cdf Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Wed, 24 Mar 2021 15:30:50 +0200 Subject: [PATCH 03/11] err handling --- cmreslogging/handlers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index a472d81..fcde1f7 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -301,12 +301,16 @@ def flush(self): } for log_record in logs_buffer ) - eshelpers.bulk( + errors = eshelpers.bulk( client=self.__get_es_client(), actions=actions, - stats_only=True, + stats_only=False, max_retries=self.max_retries, ) + if len(errors) > 0: + with open('es_logs_errors' + str(datetime.datetime.utcnow())) as errors_file: + for err in errors: + errors_file.write("%s\n" % err) except Exception as exception: if self.raise_on_indexing_exceptions: raise exception From edb5bb7f216549ad79173caa10ed149c93fdbaf4 Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Wed, 24 Mar 2021 17:47:15 +0200 Subject: [PATCH 04/11] test --- cmreslogging/handlers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index fcde1f7..16eaec7 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -301,17 +301,21 @@ def flush(self): } for log_record in logs_buffer ) - errors = eshelpers.bulk( + success_count, errors = eshelpers.bulk( client=self.__get_es_client(), actions=actions, stats_only=False, max_retries=self.max_retries, ) if len(errors) > 0: - with open('es_logs_errors' + str(datetime.datetime.utcnow())) as errors_file: + filename='es_logs_errors' + str(datetime.datetime.utcnow()) + '.log' + with open(filename, 'w') as errors_file: for err in errors: errors_file.write("%s\n" % err) except Exception as exception: + filename = 'es_logs_errors' + str(datetime.datetime.utcnow()) + '.log' + with open(filename, 'w') as errors_file: + errors_file.write("%s\n" % exception) if self.raise_on_indexing_exceptions: raise exception From b062cb3816ed2a173a1e9b15336f0c826fff77c2 Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Mon, 29 Mar 2021 10:41:44 +0300 Subject: [PATCH 05/11] error handler --- cmreslogging/handlers.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 16eaec7..83c56cc 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -280,6 +280,15 @@ def __get_es_datetime_str(timestamp): current_date = datetime.datetime.utcfromtimestamp(timestamp) return "{0!s}.{1:03d}Z".format(current_date.strftime('%Y-%m-%dT%H:%M:%S'), int(current_date.microsecond / 1000)) + def error_handler(self, exception, errors): + """ + This function can be implemented by the client to handle errors that happen when trying to write logs to ES. + Both params are optional. But one must exist. + :param exception python exception that raised while writing to ES (optional) + :param errors a list of errors returned by the bulk operation (optional) + """ + pass + def flush(self): """ Flushes the buffer into ES :return: None @@ -308,14 +317,9 @@ def flush(self): max_retries=self.max_retries, ) if len(errors) > 0: - filename='es_logs_errors' + str(datetime.datetime.utcnow()) + '.log' - with open(filename, 'w') as errors_file: - for err in errors: - errors_file.write("%s\n" % err) + self.error_handler(None, errors) except Exception as exception: - filename = 'es_logs_errors' + str(datetime.datetime.utcnow()) + '.log' - with open(filename, 'w') as errors_file: - errors_file.write("%s\n" % exception) + self.error_handler(exception, None) if self.raise_on_indexing_exceptions: raise exception From 8b9b76f4d4b2a2552b0a52e4fde4dc5a7a42933f Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Tue, 30 Mar 2021 14:16:22 +0300 Subject: [PATCH 06/11] error handling --- cmreslogging/handlers.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 83c56cc..72d9fbd 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -280,12 +280,10 @@ def __get_es_datetime_str(timestamp): current_date = datetime.datetime.utcfromtimestamp(timestamp) return "{0!s}.{1:03d}Z".format(current_date.strftime('%Y-%m-%dT%H:%M:%S'), int(current_date.microsecond / 1000)) - def error_handler(self, exception, errors): + def error_handler(self, exception): """ This function can be implemented by the client to handle errors that happen when trying to write logs to ES. - Both params are optional. But one must exist. - :param exception python exception that raised while writing to ES (optional) - :param errors a list of errors returned by the bulk operation (optional) + :param exception python exception that raised while writing to ES """ pass @@ -310,16 +308,17 @@ def flush(self): } for log_record in logs_buffer ) - success_count, errors = eshelpers.bulk( + # say we send 100 records, and get errors from 3. + # Then 97 will be uploaded successfully and an exception will be raised with a summary of the 3 errors. + # that's why we don't need the return value from the bulk. + eshelpers.bulk( client=self.__get_es_client(), actions=actions, - stats_only=False, + stats_only=True, max_retries=self.max_retries, ) - if len(errors) > 0: - self.error_handler(None, errors) except Exception as exception: - self.error_handler(exception, None) + self.error_handler(exception) if self.raise_on_indexing_exceptions: raise exception From c0d01edb77dc813e38c00407bd817f08c7a71fc2 Mon Sep 17 00:00:00 2001 From: yarden shem tov Date: Sun, 4 Apr 2021 10:57:00 +0300 Subject: [PATCH 07/11] ignore msg field --- cmreslogging/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cmreslogging/handlers.py b/cmreslogging/handlers.py index 72d9fbd..5b98825 100644 --- a/cmreslogging/handlers.py +++ b/cmreslogging/handlers.py @@ -81,6 +81,7 @@ class IndexNameFrequency(Enum): __LOGGING_FILTER_FIELDS = ['msecs', 'relativeCreated', 'levelno', + 'msg', 'created'] @staticmethod From 288e432f249de713ea2a360175a6618c79927e0b Mon Sep 17 00:00:00 2001 From: yonatanKarlik Date: Sun, 1 May 2022 10:27:18 +0300 Subject: [PATCH 08/11] removed python27 support --- requirements/requirements_py27.txt | 3 --- setup.py | 5 ----- 2 files changed, 8 deletions(-) delete mode 100644 requirements/requirements_py27.txt diff --git a/requirements/requirements_py27.txt b/requirements/requirements_py27.txt deleted file mode 100644 index 588b2f5..0000000 --- a/requirements/requirements_py27.txt +++ /dev/null @@ -1,3 +0,0 @@ -elasticsearch==5.4.0 -requests==2.18.1 -enum==0.4.6 diff --git a/setup.py b/setup.py index 08baacc..005de33 100644 --- a/setup.py +++ b/setup.py @@ -22,10 +22,6 @@ 'requests' ] -# If python version is above 3.4 (built in enums supported enums) -if sys.version_info <= (3,4): - dependencies.append('enum') - print("List of dependencies : {0}".format(str(dependencies))) setup( @@ -67,7 +63,6 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', ], From 17054ad0c1db1a04ec963d917f66a84e1e4d418d Mon Sep 17 00:00:00 2001 From: yotamis Date: Sun, 6 Nov 2022 16:39:53 +0200 Subject: [PATCH 09/11] rename --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 005de33..187e0fa 100644 --- a/setup.py +++ b/setup.py @@ -25,18 +25,18 @@ print("List of dependencies : {0}".format(str(dependencies))) setup( - name='CMRESHandler', + name='vycmreshandler', # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html version='1.0.0', - description='Elasticsearch Log handler for the logging library', + description='Voyantis fork - Elasticsearch Log handler for the logging library', long_description=long_description, # The project's main homepage. - url='https://github.com/cmanaha/python-elasticsearch-logger', + url='https://github.com/Voyantis/python-elasticsearch-logger', # Author details author='Carlos Manzanedo Rueda', From 0d44f27c208a7125081d15dfa033e20bc1be49e5 Mon Sep 17 00:00:00 2001 From: Sharon <87261624+sharontis@users.noreply.github.com> Date: Wed, 20 Sep 2023 00:23:37 +0300 Subject: [PATCH 10/11] Update version 1.0.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 187e0fa..6a3fbf1 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='1.0.0', + version='1.0.1', description='Voyantis fork - Elasticsearch Log handler for the logging library', long_description=long_description, From fcaaf67cb0a92427b61df705808f34dfcd24c5c8 Mon Sep 17 00:00:00 2001 From: vovantis Date: Tue, 4 Jun 2024 11:34:25 +0300 Subject: [PATCH 11/11] Github status checks --- .github/workflows/pre-commit.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..4f1cc30 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,11 @@ +name: Run pre-commit checks + +on: + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2