message broker agnostic (#44)

* rename topic arguments to be agnostic to broker

* added eslint and husky pre commit rule

* upgrade dependencies

* ci: 👷 Added github action to run linting on each commit

* migrated to common event store interface

* updated readme

.babelrc

@@ -0,0 +1,7 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env"
+    ]
+  ]
+}

.dockerignore

@@ -2,7 +2,7 @@
 .vscode
 node_modules
 deployment
-!deployment/dev/config/load.nginx.conf
+!deployment/config/load.nginx.conf
 docs
 www
 .env*

.env.tmpl

@@ -32,13 +32,19 @@ MONGO_URL=mongodb://localhost:27017/chat
 MONGO_USER=
 MONGO_PASSWORD=
 
-FIREBASE_PROJECT_ID=
-FIREBASE_ADMIN_CRED_JSON_PATH=
+# Redis
+REDIS_ENDPOINT=localhost:6379
+
+# Firebase
+FIREBASE_PROJECT_ID=vartalap-bef91
+FIREBASE_ADMIN_CRED_JSON_PATH=./deployment/config/firebase/service.json
 
 # s3
 S3_ACCESS_KEY_ID=
 S3_SECRET_ACCESS_KEY=
 S3_BUCKET_NAME=
 
 # Service Discovery
-SERVICE_DISCOVERY_PATH=discovery_service.json
+SERVICE_DISCOVERY_PATH=./deployment/config/discovery_service/services.json
+
+

.eslintignore

@@ -0,0 +1 @@
+node_modules/**

.eslintrc.json

@@ -1,16 +1,41 @@
 {
-    "extends": ["plugin:prettier/recommended"],
-    "env": {
-      "browser": true,
-      "es6": true,
-      "node": true
-    },
-    "plugins": ["prettier"],
-    "parserOptions": {
-      "ecmaVersion": 2020,
-      "sourceType": "module"
-    },
-    "rules": {
-      "prettier/prettier": "error"
-    }
+  "parser": "@babel/eslint-parser",
+  "extends": [
+    "airbnb",
+    "prettier",
+    "plugin:node/recommended"
+  ],
+  "plugins": [
+    "prettier"
+  ],
+  "env": {
+    "node": true,
+    "es2021": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 12,
+    "sourceType": "module"
+  },
+  "rules": {
+    "no-console": "warn",
+    "consistent-return": "off",
+    "no-process-exit": "off",
+    "no-param-reassign": "off",
+    "no-return-await": "off",
+    "no-underscore-dangle": "off",
+    "prefer-destructuring": [
+      "error",
+      {
+        "object": true,
+        "array": false
+      }
+    ],
+    "no-unused-vars": [
+      "error",
+      {
+        "ignoreRestSiblings": true,
+        "argsIgnorePattern": "^_"
+      }
+    ]
   }
+}

.github/workflows/lint.yml

@@ -0,0 +1,21 @@
+name: "Linting"
+
+on:
+  push:
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Dependencies
+        run: yarn install --immutable
+
+      - name: Run liniting
+        run: npm run lint

.gitignore

@@ -115,5 +115,8 @@ dist
 .yarn/install-state.gz
 .pnp.*
 
+# Redis dump file
+*.rdb
+
 kafka
-*.local.*
+*.local.*

.gitpod.yml

@@ -1,26 +1,27 @@
 
 image:
-  file: .gitpod.Dockerfile
+  file: .gitpod/Dockerfile
 
 ports:
   - port: 8080
     onOpen: open-browser
 
 tasks:
   - init: yarn install
-  - name: Kafka 
+  - name: Kafka
+    init: chmod +x deployment/scripts/*.bash && ./deployment/scripts/setup.bash 
     command: > 
-      chmod +x deployment/dev/scripts/init-kafka.bash &&
-      cp .env.tmpl .env &&
       KAFKA_HOME=/kafka &&
-      ./deployment/dev/scripts/init-kafka.bash $KAFKA_HOME .env
+      ./deployment/scripts/init-kafka.bash $KAFKA_HOME .env
   - name: Nginx
-    command: nginx
+    command: export NGINX_DOCROOT="${GITPOD_REPO_ROOT}/www" && nginx
   - name: mongodb
     command: mkdir -p /workspace/data && mongod --dbpath /workspace/data
   - name: redis
     command:  redis-server 
 
+
 vscode:
   extensions:
     - editorconfig.editorconfig
+    - dbaeumer.vscode-eslint

.gitpod.Dockerfile → .gitpod/Dockerfile

@@ -1,12 +1,10 @@
 FROM gitpod/workspace-mongodb
 
-COPY ./deployment/dev/config/load.nginx.conf /etc/nginx/nginx.conf
+COPY .gitpod/load.nginx.conf /etc/nginx/nginx.conf
 
 RUN sudo apt-get update && sudo apt-get install redis-server -y
 
 RUN wget https://downloads.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz  && \
     tar xzf kafka_2.13-3.0.0.tgz && \
     sudo mv kafka_2.13-3.0.0 /kafka && \
     rm kafka_2.13-3.0.0.tgz
-
-ENV NGINX_DOCROOT_IN_REPO="www"

deployment/dev/config/load.nginx.conf → .gitpod/load.nginx.conf

@@ -1,4 +1,4 @@
 # This is file will load the main nginx.conf from this directory, This is done as workaround to enable editing of nginx.conf while working
 # as gitpod doesn't allow you to edit nginx.conf once it pushed into the /var/nginx/nginx.conf
-
-include /workspace/chat-server/deployment/dev/config/nginx.conf;
+env GITPOD_REPO_ROOT;
+include /workspace/chat-server/deployment/config/nginx/nginx.dev.conf;

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npm run lint

.vscode/launch.json

@@ -12,9 +12,10 @@
         "--app-name=ws-gateway-1",
         "--debug",
         "--gateway-name=gateway-1",
+        "--event-store=kafka",
         "--kafka-client-id=gateway-1",
-        "--kafka-user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
-        "--kafka-new-message-topic=${TOPIC_NEW_MESSAGE}",
+        "--user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
+        "--new-message-topic=${TOPIC_NEW_MESSAGE}",
         "--kafka-broker-list=${BROKER_LIST}",
         "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
         "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
@@ -32,12 +33,13 @@
         "--app-name=message-delivery",
         "--debug",
         "--mongo-url=${MONGO_URL}",
+        "--event-store=kafka",
         "--kafka-client-id=message-delivery-1",
         "--kafka-consumer-group=${CONSUMER_GROUP_MESSAGE_DELIVERY}",
-        "--kafka-user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
-        "--kafka-send-message-topic=${TOPIC_SEND_MESSAGE}",
-        "--kafka-ack-topic=${TOPIC_ACK_MESSAGE}",
-        "--kafka-offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
+        "--user-connection-state-topic=${TOPIC_USER_CONNECTION_STATE}",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--ack-topic=${TOPIC_ACK_MESSAGE}",
+        "--offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
         "--service-discovery-path=${SERVICE_DISCOVERY_PATH}",
         "--kafka-broker-list=${BROKER_LIST}",
         "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
@@ -57,13 +59,14 @@
       "args": [
         "--app-name=message-router-1",
         "--debug",
+        "--event-store=kafka",
         "--kafka-client-id=message-router-1",
         "--kafka-consumer-group=${CONSUMER_GROUP_MESSAGE_ROUTER}",
-        "--kafka-send-message-topic=${TOPIC_SEND_MESSAGE}",
-        "--kafka-new-message-topic=${TOPIC_NEW_MESSAGE}",
-        "--kafka-ack-topic=${TOPIC_ACK_MESSAGE}",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--new-message-topic=${TOPIC_NEW_MESSAGE}",
+        "--ack-topic=${TOPIC_ACK_MESSAGE}",
+        "--group-message-topic=${TOPIC_NEW_GROUP_MS}",
         "--kafka-broker-list=${BROKER_LIST}",
-        "--kafka-group-message-topic=${TOPIC_NEW_GROUP_MS}",
         "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
         "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
         "--kafka-sasl-password=${KAFKA_SASL_PASSWORD}"
@@ -82,11 +85,12 @@
         "--port=4000",
         "--mongo-url=${MONGO_URL}",
         "--firebase-project-id=${FIREBASE_PROJECT_ID}",
+        "--event-store=kafka",
         "--kafka-broker-list=${BROKER_LIST}",
         "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
         "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
         "--kafka-sasl-password=${KAFKA_SASL_PASSWORD}",
-        "--kafka-new-login-topic=${TOPIC_NEW_LOGIN}"
+        "--new-login-topic=${TOPIC_NEW_LOGIN}"
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
@@ -119,10 +123,10 @@
         "--debug",
         "--port=4002",
         "--mongo-url=mongodb://localhost:27017/chat",
-        "--json-server-port=${GROUP_MS_JSON_PORT}",
+        "--event-store=kafka",
         "--kafka-client-id=group-1",
         "--kafka-broker-list=${BROKER_LIST}",
-        "--kafka-new-group-message-topic=${TOPIC_NEW_GROUP_MS}"
+        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}"
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
@@ -135,14 +139,15 @@
       "args": [
         "--app-name=group-message-router-1",
         "--debug",
+        "--event-store=kafka",
         "--kafka-client-id=group-message-router-1",
-        "--kafka-send-message-topic=${TOPIC_SEND_MESSAGE}",
+        "--send-message-topic=${TOPIC_SEND_MESSAGE}",
         "--kafka-consumer-group=${CONSUMER_GROUP_GROUP_MESSAGE_ROUTER}",
-        "--kafka-new-group-message-topic=${TOPIC_NEW_GROUP_MS}",
+        "--new-group-message-topic=${TOPIC_NEW_GROUP_MS}",
         "--kafka-ack-topic=${TOPIC_ACK_MESSAGE}",
         "--kafka-broker-list=${BROKER_LIST}",
         "--mongo-url=${MONGO_URL}",
-        "--kafka-persistence-message-topic=${TOPIC_DB_MESSAGE}"
+        "--persistence-message-topic=${TOPIC_DB_MESSAGE}"
       ],
       "envFile": "${workspaceFolder}/.env",
       "autoAttachChildProcesses": true
@@ -157,13 +162,14 @@
         "--debug",
         "--mongo-url=${MONGO_URL}",
         "--firebase-admin-credential-json-path=${FIREBASE_ADMIN_CRED_JSON_PATH}",
+        "--event-store=kafka",
         "--kafka-broker-list=${BROKER_LIST}",
         "--kafka-security-protocol=${KAFKA_SECURITY_PROTOCOL}",
         "--kafka-sasl-username=${KAFKA_SASL_USERNAME}",
         "--kafka-sasl-password=${KAFKA_SASL_PASSWORD}",
         "--kafka-consumer-group=notification-ms",
-        "--kafka-new-login-topic=${TOPIC_NEW_LOGIN}",
-        "--kafka-offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
+        "--new-login-topic=${TOPIC_NEW_LOGIN}",
+        "--offline-message-topic=${TOPIC_OFFLINE_MESSAGE}",
         "--db-app-initial=${PERSISTENCE_MESSAGE_MS_INITIAL}"
       ],
       "envFile": "${workspaceFolder}/.env",

README.md

@@ -107,14 +107,16 @@ A chat server based on the microservice architecture to ensure high availability
 
 | Name |  Description |
 | ---- |  ----------- |
-| deployment | Deployment scripts, configurations for dev as well as prod |
+| .github | Configuration files related to github like workflows, funding |
+| .gipod | Configuration files for gitpod dev enviroment setup |
+| .husky | git hooks configuration via [husky](https://typicode.github.io/husky) |
+| deployment | Deployment scripts |
 | docs | Docs related to the project |
 | helper | Common utility functions |
 | libs | Project specific libs like base class for microservice, resource configuration and initialization |
 | services | Microservice |
 | www | Web based testing interface |
 | .env.tmpl | Environment template file |
-| .gitpod.Dockerfile | Docker file for gitpod setup |
 | .gitpod.yml | Gitpod configuration file |
 | LICENSE | Project License file (MIT) |
 | package.json | Node project configuration file |
@@ -153,15 +155,18 @@ Click on the Gitpod badge it will start the fully setup development environment.
     ```
 - Initialize Kafka 
     ```
-    cd deployment/dev/scripts
+    cd deployment/scripts
     ./init-kafka.bash ${KAFKA_INSTALLATION_DIRECTORY} .env
     ```
 - Open project in vscode
 - Start the required microservice from `RUN and DEBUG` option
-- (Optional) Start nginx using the configuration [deployment/dev/config/nginx.config](./deployment/dev/config/nginx.config)
+- (Optional) Start nginx using the configuration [deployment/config/nginx.config](./deployment/config/nginx.dev.config)
 
 
 
+
+## Deployment
+For deployment guide refer to [deployment/README.md](./deployment/README.md).
 # Resources
 To follow the update keep a eye on vartalap blogs on [blog.one9x.org](https://blog.one9x.org)
 Some of the relevent blogs are:

deployment/.env.tmpl

@@ -0,0 +1,46 @@
+# Infra
+STAGE=dev
+CHAT_SERVER_TAG=v2.2.6
+CONFIG_DIR=./config
+
+# KAFKA TOPIC
+TOPIC_USER_CONNECTION_STATE=user-connection-state
+TOPIC_SEND_MESSAGE=send-message
+TOPIC_NEW_MESSAGE=new-message
+TOPIC_OFFLINE_MESSAGE=offline-message
+TOPIC_NEW_GROUP_MS=new-group-message
+TOPIC_NEW_LOGIN=new-login
+TOPIC_ACK_MESSAGE=ack
+
+# CREATE TOPIC FOR NAMED SERVICE
+PERSISTENCE_MESSAGE_MS_INITIAL=persistence-message
+
+# KAFKA OPTIONS
+BROKER_LIST=localhost:9092
+ZOOKEEPER_ENDPOINT=localhost:2181
+KAFKA_SECURITY_PROTOCOL=
+KAFKA_SASL_USERNAME=
+KAFKA_SASL_PASSWORD=
+
+# KAFKA CONSUMER GROUP
+CONSUMER_GROUP_MESSAGE_ROUTER=message-router
+CONSUMER_GROUP_MESSAGE_DELIVERY=message-delivery
+CONSUMER_GROUP_GATEWAY=gateway
+CONSUMER_GROUP_GROUP_MESSAGE_ROUTER=group-message-router
+
+
+# MONGO URL
+MONGO_URL=mongodb://localhost:27017/chat
+MONGO_USER=
+MONGO_PASSWORD=
+
+# Redis
+REDIS_ENDPOINT=localhost:6379
+
+# Firebase
+FIREBASE_PROJECT_ID=vartalap-bef91
+
+# s3
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET_NAME=

deployment/.gitignore

@@ -0,0 +1 @@
+*.local.*