feat add support for nats (#84)

* feat: add support for nats jetstream as event store
ramank775 · Jul 31, 2022 · c966e5f · c966e5f
1 parent c1c9b9b
commit c966e5f
Show file tree

Hide file tree

Showing 13 changed files with 547 additions and 60 deletions.
diff --git a/.env.tmpl b/.env.tmpl
@@ -57,4 +57,5 @@ S3_BUCKET_NAME=
 # Service Discovery
 SERVICE_DISCOVERY_PATH=./deployment/config/discovery_service/services.json
 
-
+# NATS
+NATS_SERVER_LIST=nats://127.0.0.1:4222
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,4 +1,3 @@
-
 image:
   file: .gitpod/Dockerfile
 
@@ -9,17 +8,19 @@ ports:
 tasks:
   - init: yarn install
   - name: Kafka
-    init: chmod +x deployment/scripts/*.bash && ./deployment/scripts/setup.bash 
-    command: > 
+    init: chmod +x deployment/scripts/*.bash && ./deployment/scripts/setup.bash
+    command: >
       KAFKA_HOME=/kafka &&
       ./deployment/scripts/init-kafka.bash $KAFKA_HOME .env
+  - name: Nats
+    init: chmod +x deployment/scripts/*.bash && ./deployment/scripts/setup.bash
+    command: ./deployment/scripts/init-nats.bash .env
   - name: Nginx
     command: export NGINX_DOCROOT="${GITPOD_REPO_ROOT}/www" && nginx
   - name: mongodb
     command: mkdir -p /workspace/data && mongod --dbpath /workspace/data
   - name: redis
-    command:  redis-server 
-
+    command: redis-server
 
 vscode:
   extensions:

diff --git a/.gitpod/Dockerfile b/.gitpod/Dockerfile
@@ -1,10 +1,24 @@
 FROM gitpod/workspace-mongodb
 
+ARG NATS_VERSION="v2.8.3"
+ARG KAFKA_VERSION="3.2.0"
+ARG NATS_CLI_VERSION="0.0.33"
+
 COPY .gitpod/load.nginx.conf /etc/nginx/nginx.conf
 
 RUN sudo apt-get update && sudo apt-get install redis-server -y
 
-RUN wget https://downloads.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz  && \
-    tar xzf kafka_2.13-3.0.0.tgz && \
-    sudo mv kafka_2.13-3.0.0 /kafka && \
-    rm kafka_2.13-3.0.0.tgz
+RUN wget https://downloads.apache.org/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz  && \
+  tar xzf kafka_2.13-${KAFKA_VERSION}.tgz && \
+  sudo mv kafka_2.13-${KAFKA_VERSION} /kafka && \
+  rm kafka_2.13-${KAFKA_VERSION}.tgz
+
+RUN curl -L https://github.com/nats-io/nats-server/releases/download/${NATS_VERSION}/nats-server-${NATS_VERSION}-linux-amd64.zip -o nats-server.zip && \
+  unzip nats-server.zip -d nats-server && \
+  sudo cp nats-server/nats-server-${NATS_VERSION}-linux-amd64/nats-server /usr/bin && \
+  rm -rf nats-server.zip nats-server
+
+RUN curl -L https://github.com/nats-io/natscli/releases/download/v${NATS_CLI_VERSION}/nats-${NATS_CLI_VERSION}-linux-amd64.zip -o nats-cli.zip && \
+  unzip nats-cli.zip -d nats-cli && \
+  sudo cp nats-cli/nats-${NATS_CLI_VERSION}-linux-amd64/nats /usr/bin && \
+  rm -rf nats-cli.zip nats-cli
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -6,7 +6,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Gateway MS",
+      "name": "Launch Gateway MS - Kafka",
       "program": "${workspaceFolder}/services/connection-gateway/gateway.js",
       "args": [
         "--app-name=ws-gateway-1",
@@ -27,7 +27,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Message Delivery MS",
+      "name": "Launch Message Delivery MS - Kafka",
       "program": "${workspaceFolder}/services/message-delivery/message-delivery-ms.js",
       "args": [
         "--app-name=message-delivery",
@@ -54,7 +54,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Message Router MS",
+      "name": "Launch Message Router MS - Kafka",
       "program": "${workspaceFolder}/services/message-router-ms/message-router-ms.js",
       "args": [
         "--app-name=message-router-1",
@@ -77,7 +77,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch profile MS",
+      "name": "Launch profile MS - Kafka",
       "program": "${workspaceFolder}/services/profile-ms/profile-ms.js",
       "args": [
         "--app-name=profile-ms-1",
@@ -99,7 +99,70 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch file MS",
+      "name": "Launch group MS - Kafka",
+      "program": "${workspaceFolder}/services/group-ms/group-ms.js",
+      "args": [
+        "--app-name=group-ms-1",
+        "--debug",
+        "--port=4002",
+        "--mongo-url=mongodb://localhost:27017/chat",
+        "--event-store=kafka",
+        "--kafka-client-id=group-1",
+        "--kafka-broker-list=${BROKER_LIST}",
+        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}"
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Group Message Router MS - Kafka",
+      "program": "${workspaceFolder}/services/group-ms/group-message-router-ms.js",
+      "args": [
+        "--app-name=group-message-router-1",
+        "--debug",
+        "--event-store=kafka",
+        "--kafka-client-id=group-message-router-1",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--kafka-consumer-group=${CONSUMER_GROUP_GROUP_MESSAGE_ROUTER}",
+        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}",
+        "--system-message-topic=${TOPIC_SYSTEM_MESSAGE}",
+        "--kafka-broker-list=${BROKER_LIST}",
+        "--mongo-url=${MONGO_URL}",
+        "--persistence-message-topic=${TOPIC_DB_MESSAGE}"
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch notification MS - Kafka",
+      "program": "${workspaceFolder}/services/notification-ms/notification-ms.js",
+      "args": [
+        "--app-name=notification-ms-1",
+        "--debug",
+        "--mongo-url=${MONGO_URL}",
+        "--pn-service=mock",
+        "--firebase-admin-credential-json-path=${workspaceFolder}/${FIREBASE_ADMIN_CRED_JSON_PATH}",
+        "--event-store=kafka",
+        "--kafka-broker-list=${BROKER_LIST}",
+        "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
+        "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
+        "--kafka-sasl-password=${KAFKA_SASL_PASSWORD}",
+        "--kafka-consumer-group=notification-ms",
+        "--new-login-topic=${TOPIC_NEW_LOGIN}",
+        "--offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
+        "--db-app-initial=${PERSISTENCE_MESSAGE_MS_INITIAL}"
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch file MS - Common",
       "program": "${workspaceFolder}/services/file-ms/file-ms.js",
       "args": [
         "--app-name=file-ms-1",
@@ -119,65 +182,138 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch group MS",
+      "name": "Launch Gateway MS - Nats",
+      "program": "${workspaceFolder}/services/connection-gateway/gateway.js",
+      "args": [
+        "--app-name=ws-gateway-1",
+        "--debug",
+        "--gateway-name=gateway-1",
+        "--event-store=nats",
+        "--user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
+        "--new-message-topic=${TOPIC_NEW_MESSAGE}",
+        "--nats-server-list=${NATS_SERVER_LIST}"
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true,
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Message Delivery MS - NATS",
+      "program": "${workspaceFolder}/services/message-delivery/message-delivery-ms.js",
+      "args": [
+        "--app-name=message-delivery",
+        "--debug",
+        "--mongo-url=${MONGO_URL}",
+        "--event-store=nats",
+        "--user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--system-message-topic=${TOPIC_SYSTEM_MESSAGE}",
+        "--offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
+        "--service-discovery-path=${SERVICE_DISCOVERY_PATH}",
+        "--nats-server-list=${NATS_SERVER_LIST}",
+        "--nats-consumer-group=${CONSUMER_GROUP_MESSAGE_DELIVERY}",
+        "--cache-type=redis",
+        "--redis-endpoint=127.0.0.1:6379"
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Message Router MS - NATS",
+      "program": "${workspaceFolder}/services/message-router-ms/message-router-ms.js",
+      "args": [
+        "--app-name=message-router-1",
+        "--debug",
+        "--event-store=nats",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--new-message-topic=${TOPIC_NEW_MESSAGE}",
+        "--system-message-topic=${TOPIC_SYSTEM_MESSAGE}",
+        "--group-message-topic=${TOPIC_NEW_GROUP_MS}",
+        "--nats-server-list=${NATS_SERVER_LIST}",
+        "--nats-consumer-group=${CONSUMER_GROUP_MESSAGE_ROUTER}",
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch profile MS - NATS",
+      "program": "${workspaceFolder}/services/profile-ms/profile-ms.js",
+      "args": [
+        "--app-name=profile-ms-1",
+        "--debug",
+        "--port=4000",
+        "--auth-provider=mock",
+        "--mongo-url=${MONGO_URL}",
+        "--firebase-project-id=${FIREBASE_PROJECT_ID}",
+        "--event-store=nats",
+        "--new-login-topic=${TOPIC_NEW_LOGIN}",
+        "--nats-server-list=${NATS_SERVER_LIST}",
+      ],
+      "envFile": "${workspaceFolder}/.env",
+      "autoAttachChildProcesses": true
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch group MS - NATS",
       "program": "${workspaceFolder}/services/group-ms/group-ms.js",
       "args": [
         "--app-name=group-ms-1",
         "--debug",
         "--port=4002",
         "--mongo-url=mongodb://localhost:27017/chat",
-        "--event-store=kafka",
-        "--kafka-client-id=group-1",
-        "--kafka-broker-list=${BROKER_LIST}",
-        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}"
+        "--event-store=nats",
+        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}",
+        "--nats-server-list=${NATS_SERVER_LIST}",
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Group Message Router MS",
+      "name": "Launch Group Message Router MS - NATS",
       "program": "${workspaceFolder}/services/group-ms/group-message-router-ms.js",
       "args": [
         "--app-name=group-message-router-1",
         "--debug",
-        "--event-store=kafka",
-        "--kafka-client-id=group-message-router-1",
+        "--event-store=nats",
         "--send-message-topic=${TOPIC_SEND_MESSAGE}",
-        "--kafka-consumer-group=${CONSUMER_GROUP_GROUP_MESSAGE_ROUTER}",
         "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}",
         "--system-message-topic=${TOPIC_SYSTEM_MESSAGE}",
-        "--kafka-broker-list=${BROKER_LIST}",
+        "--persistence-message-topic=${TOPIC_DB_MESSAGE}",
+        "--nats-server-list=${NATS_SERVER_LIST}",
+        "--nats-consumer-group=${CONSUMER_GROUP_GROUP_MESSAGE_ROUTER}",
         "--mongo-url=${MONGO_URL}",
-        "--persistence-message-topic=${TOPIC_DB_MESSAGE}"
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
     },
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch notification MS",
+      "name": "Launch notification MS - NATS",
       "program": "${workspaceFolder}/services/notification-ms/notification-ms.js",
       "args": [
         "--app-name=notification-ms-1",
         "--debug",
         "--mongo-url=${MONGO_URL}",
         "--pn-service=mock",
         "--firebase-admin-credential-json-path=${workspaceFolder}/${FIREBASE_ADMIN_CRED_JSON_PATH}",
-        "--event-store=kafka",
-        "--kafka-broker-list=${BROKER_LIST}",
-        "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
-        "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
-        "--kafka-sasl-password=${KAFKA_SASL_PASSWORD}",
-        "--kafka-consumer-group=notification-ms",
+        "--event-store=nats",
+        "--nats-server-list=${NATS_SERVER_LIST}",
+        "--nats-consumer-group=notification-ms",
         "--new-login-topic=${TOPIC_NEW_LOGIN}",
         "--offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
         "--db-app-initial=${PERSISTENCE_MESSAGE_MS_INITIAL}"
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
-    }
+    },
   ]
 }
diff --git a/deployment/.env.tmpl b/deployment/.env.tmpl
@@ -2,6 +2,7 @@
 STAGE=dev
 CHAT_SERVER_TAG=v2.2.6
 CONFIG_DIR=./config
+EVENT_STORE=kafka
 
 # KAFKA TOPIC
 TOPIC_USER_CONNECTION_STATE=user-connection-state
@@ -31,7 +32,7 @@ KAFKA_SECURITY_PROTOCOL=
 KAFKA_SASL_USERNAME=
 KAFKA_SASL_PASSWORD=
 
-# KAFKA CONSUMER GROUP
+# EVENT STORE CONSUMER GROUP
 CONSUMER_GROUP_MESSAGE_ROUTER=message-router
 CONSUMER_GROUP_MESSAGE_DELIVERY=message-delivery
 CONSUMER_GROUP_GATEWAY=gateway
@@ -54,3 +55,12 @@ PN_TTL=3600
 S3_ACCESS_KEY_ID=
 S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
+
+# NATS
+NATS_SERVER_LIST=nats://127.0.0.1:4222
+NATS_AUTH=
+NATS_USER=
+NATS_PASS=
+NATS_AUTH_TOKEN=
+NATS_NKEY=
+NATS_USER_JWT=