From c445c2a951e13ed18fa527cd7eb9695b60c8d68f Mon Sep 17 00:00:00 2001 From: Jonathan Rivers Date: Wed, 10 Nov 2021 12:41:21 -0500 Subject: [PATCH 1/3] give the client the ability to specify the consumer group id --- README.md | 13 +++++++++++++ .../ncdsclient/consumer/NasdaqKafkaAvroConsumer.py | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a91cb8..52b412f 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,19 @@ Replace example stream properties in the file **kafka-config.json** (https://git For optional consumer configurations see: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + The consumer configuration gives the ability to fix a consumer group id which is important for production usage. Setting the consumer group id explicitly allows the kafka administrator to add protections around your production feed. + The following is an example of a fixed consumer group id convention + ```properties + group.id: {oauth.client.id}_{environment}_{feedname} + ``` + The following is an example of the group id filled in: + ```properties + group.id: testuser_pro_totalview + ``` + Please note that if you fix the consumer group id in your client, you will continue at the last offset that was consumed from kafka. This is a way promote availability for your application as the app will begin where it left off given a disconnection. + + Also, if you are using the SDK for interactive query, you will also be consuming from where you last consumed. The consumer group id effectively saves the last offset that you have ingested. + ### Client Authentication configuration Replace example client authentication properties in the file **client-authentication-config.json** (https://github.com/Nasdaq/NasdaqCloudDataService-SDK-Python/blob/master/ncdssdk_client/src/main/python/resources/client-authentication-config.json) with valid credentials provided during on-boarding. diff --git a/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py b/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py index 5c1e932..6440a09 100644 --- a/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py +++ b/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py @@ -132,7 +132,8 @@ def get_consumer(self, avro_schema, stream_name): """ if 'auto.offset.reset' not in self.kafka_props: self.kafka_props[config.AUTO_OFFSET_RESET_CONFIG] = "earliest" - self.kafka_props[config.GROUP_ID_CONFIG] = f'{self.client_ID}_{stream_name}_{datetime.datetime.today().day}' + if 'group_id' not in self.kafka_props: + self.kafka_props[config.GROUP_ID_CONFIG] = f'{self.client_ID}_{stream_name}_{datetime.datetime.today().day}' return KafkaAvroConsumer(self.kafka_props, avro_schema) def get_schema_for_topic(self, topic): From da557bd9f12ecf741a0d3a0e3171f74e4463fdbd Mon Sep 17 00:00:00 2001 From: Jonathan Rivers Date: Fri, 12 Nov 2021 09:30:01 -0500 Subject: [PATCH 2/3] fixed - to . ( bug ) --- .../main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py b/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py index 6440a09..20d10b3 100644 --- a/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py +++ b/ncdssdk/src/main/python/ncdsclient/consumer/NasdaqKafkaAvroConsumer.py @@ -132,7 +132,7 @@ def get_consumer(self, avro_schema, stream_name): """ if 'auto.offset.reset' not in self.kafka_props: self.kafka_props[config.AUTO_OFFSET_RESET_CONFIG] = "earliest" - if 'group_id' not in self.kafka_props: + if 'group.id' not in self.kafka_props: self.kafka_props[config.GROUP_ID_CONFIG] = f'{self.client_ID}_{stream_name}_{datetime.datetime.today().day}' return KafkaAvroConsumer(self.kafka_props, avro_schema) From f9cd77020a921ae53af5fee103fd04cfc4769ce6 Mon Sep 17 00:00:00 2001 From: Leonid Rosenfeld Date: Mon, 29 Nov 2021 17:01:28 -0500 Subject: [PATCH 3/3] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 52b412f..00fa4af 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Nasdaq Cloud Data Service (NCDS) provides a modern and efficient method of deliv ### Equities #### The Nasdaq Stock Market - [Nasdaq Basic](http://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NasdaqBasic-Cloud.pdf) -- [Nasdaq Last Sale+](http://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NLSPlus-cloud.pdf) +- [Nasdaq Last Sale+](http://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NLSPlus-cloud.pdf) (real-time & delayed) - [Nasdaq TotalView](http://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/Totalview-ITCH-cloud.pdf) - [Nasdaq Consolidated Quotes and Trades](https://github.com/Nasdaq/CloudDataService/raw/master/specs/CQT-cloud.pdf) #### Nasdaq BX