Configured VPA script

.gitignore

@@ -3,7 +3,7 @@
 .idea
 .classpath
 *.DS_Store
-*.terrahelpbkp 
+*.terrahelpbkp
 **/.terraform/*
 automation-scripts/infra-setup/awscliv2.zip
 automation-scripts/infra-setup/aws/dist
@@ -19,4 +19,14 @@ automation-scripts/infra-setup/aws/THIRD_PARTY_LICENSES
 # Ignore everything in the helmcharts/kitchen directory
 helmcharts/kitchen/*
 # Exclude install.sh from being ignored
-!helmcharts/kitchen/install.sh
+!helmcharts/kitchen/install.sh
+node_modules
+package-lock.json
+
+Ignore the global-key-values.yaml file as it is automatically created by Terraform
+terraform/aws/global-values.yaml
+terraform/aws/private_key.pem
+terraform/aws/public_key.pem
+*.tfvars
+*.env
+.env

Dockerfiles/flink-connectors/Dockerfile

@@ -1,4 +1,4 @@
-FROM flink:1.17.2-scala_2.12-java11 AS base_image
+FROM flink:1.17.2-scala_2.12-java11
 
 USER root
 RUN apt update -y
@@ -14,7 +14,4 @@ RUN cd $FLINK_HOME/plugins/s3-fs-presto && curl -LO https://repo1.maven.org/mave
 RUN cd $FLINK_HOME/plugins/gs-fs-hadoop && curl -LO https://repo1.maven.org/maven2/org/apache/flink/flink-gs-fs-hadoop/1.17.2/flink-gs-fs-hadoop-1.17.2.jar
 
 COPY ./connectors /data/connectors-init
-RUN pip install -r /data/connectors-init/requirements.txt
-
-FROM base_image AS kafka_connector_image
-COPY kafka-connector-1.0.0-distribution.tar.gz /tmp/obsrv-connectors-extracted/
+RUN pip install -r /data/connectors-init/requirements.txt

Dockerfiles/kubectl/Dockerfile

@@ -0,0 +1,2 @@
+FROM alpine:3
+RUN apk add --no-cache curl kubectl>=1.32.0-r1 jq openssl postgresql-client

Dockerfiles/postgresql-backup/Dockerfile

@@ -1,15 +1,15 @@
-FROM --platform=linux/amd64 python:alpine
+FROM python:alpine
 COPY ./postgres-backup.sh /
-RUN apk --no-cache add postgresql16-client && apk add --no-cache aws-cli
+RUN apk --no-cache add postgresql17-client && apk add --no-cache aws-cli
 
 RUN apk add --no-cache curl py3-pip
 RUN apk add --no-cache gcc musl-dev python3-dev libffi-dev openssl-dev cargo make
 RUN pip install --upgrade pip
 RUN pip install azure-cli
-RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-471.0.0-linux-x86_64.tar.gz
-RUN tar -xf /google-cloud-cli-471.0.0-linux-x86_64.tar.gz
+RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz
+RUN tar -xf /google-cloud-cli-linux-x86_64.tar.gz
 RUN /google-cloud-sdk/install.sh --quiet
-RUN rm /google-cloud-cli-471.0.0-linux-x86_64.tar.gz
+RUN rm /google-cloud-cli-linux-x86_64.tar.gz
 RUN /google-cloud-sdk/bin/gcloud config set disable_usage_reporting true
 RUN apk del curl py3-pip cargo make
 

Dockerfiles/superset/Dockerfile

@@ -0,0 +1,4 @@
+FROM apache/superset:4.1.1
+USER root
+RUN pip install Authlib psycopg2-binary pydruid sqlalchemy-trino
+USER superset

connectors/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+*.gz
+*.zip
+*.tar

connectors/README.md

@@ -0,0 +1,9 @@
+# Connectors
+
+To register the connectors, place the packages under `distributions` folder.
+
+Once the distributions are placed, run
+
+```bash
+./register.sh
+```

connectors/register.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+dataset_api_port=3000
+
+# function to check and install kubectl
+install_kubectl() {
+    if ! command -v kubectl &> /dev/null; then
+        echo "kubectl is not installed. Would you like to install kubectl? (yes/no)"
+        read -r response
+        if [ "$response" == "yes" ]; then
+
+            ### Linux x86_64
+            if [ "$(uname -s)" == "Linux" ] && [ "$(uname -m)" == "x86_64" ]; then
+                echo "Installing kubectl..."
+                   curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+                chmod +x kubectl
+                sudo mv kubectl /usr/local/bin/
+                echo "kubectl has been successfully installed."
+            ## Linux arm64
+            elif [ "$(uname -s)" == "Linux" ] && [ "$(uname -m)" == "aarch64" ]; then
+                curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
+                chmod +x kubectl
+                sudo mv kubectl /usr/local/bin/
+                echo "kubectl has been successfully installed."
+            ## Mac
+            elif [ "$(uname -s)" == "Darwin" ]; then
+                curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
+                chmod +x kubectl
+                sudo mv kubectl /usr/local/bin/
+                echo "kubectl has been successfully installed."
+            else
+                echo "Unable to detect supported OS. Please install kubectl manually and re-run the script."
+                exit 1
+            fi
+        else
+            echo "kubectl is required to proceed. Please install kubectl and re-run the script."
+            exit 1
+        fi
+    else
+        echo "kubectl is already installed."
+    fi
+}
+
+
+check_kubeconfig() {
+    if [ -z "$KUBECONFIG" ]; then
+        echo "KUBECONFIG is not set and isrequired to proceed. Please set KUBECONFIG and re-run the script."
+        exit 1
+    else
+        echo "Using KUBECONFIG from '$KUBECONFIG' and running with current-context: '$(kubectl config current-context)'"
+    fi
+}
+
+check_dataset_installation(){
+    dataset_api_pod=$(kubectl get pods -n dataset-api --selector app.kubernetes.io/name=dataset-api -o jsonpath='{.items[0].metadata.name}')
+
+    # check if dataset api pod starts with dataset-api
+    if [[ $dataset_api_pod == "dataset-api"* ]]; then
+        echo "Dataset API is installed [$dataset_api_pod]"
+    else
+        echo "Dataset API is not installed. Please install Dataset API before proceeding."
+        exit 1
+    fi
+}
+
+open_dataset_api_ports(){
+    echo "Opening ports for dataset api..."
+    # generate random port number and see if its available
+    while [ $dataset_api_port -lt 65535 ]; do
+        if ! lsof -i:$dataset_api_port; then
+            echo "Port $dataset_api_port is available."
+            break
+        else
+            dataset_api_port=$((RANDOM % 65535))
+        fi
+    done
+
+    # Open ports for dataset api in background
+    kubectl port-forward -n dataset-api $dataset_api_pod $dataset_api_port:3000 &
+
+    # Wait for port-forward to start
+    number_of_tries=0
+    while ! lsof -i:$dataset_api_port; do
+        echo "Waiting for port-forward to start..."
+        sleep 10
+        number_of_tries=$((number_of_tries + 1))
+        if [ $number_of_tries -eq 10 ]; then
+            echo "Failed to start port-forward. Please check the logs and try again."
+            exit 1
+        fi
+    done
+    echo "Dataset API is now accessible at http://localhost:$dataset_api_port"
+}
+
+register_connectors() {
+    # list all normal files in distributions directory
+    for connector in $(ls -1 distributions/*.tar.gz); do
+        echo "\nRegistering connector: $connector"
+        curl --progress-bar --location "localhost:$dataset_api_port/v2/connector/register" \
+        --header 'Content-Type: multipart/form-data' \
+        --form "file=@$connector" | cat
+        rm -rf $connector
+    done
+}
+
+close_dataset_api_ports(){
+    # Close port-forward
+    kill $(lsof -t -i:$dataset_api_port)
+    echo "Port-forward is closed."
+}
+
+install_kubectl
+check_kubeconfig
+check_dataset_installation
+open_dataset_api_ports
+register_connectors
+close_dataset_api_ports
+
+## Handle SIGINT and SIGTERM and close port-forward
+trap 'close_dataset_api_ports' SIGINT SIGTERM

exporters/kafka-message-exporter/.dockerignore

@@ -0,0 +1 @@
+node_modules

exporters/kafka-message-exporter/Dockerfile

@@ -1,4 +1,6 @@
-FROM scratch
-ADD kafka-message-exporter /
-EXPOSE 8000
-CMD ["/kafka-message-exporter"]
+FROM --platform=linux/amd64 node:23.4-alpine
+WORKDIR /opt/app
+COPY ./package.json .
+RUN npm install --silent
+COPY . .
+CMD ["npm", "run", "start"]

exporters/kafka-message-exporter/app.js

@@ -0,0 +1,28 @@
+var createError = require('http-errors');
+var express = require('express');
+var path = require('path');
+var cookieParser = require('cookie-parser');
+
+var indexRouter = require('./routes/index');
+const { createConsumers } = require('./services/kafka');
+
+var app = express();
+
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+
+createConsumers();
+
+app.use('/', indexRouter);
+
+app.use(function (req, res, next) {
+  next(createError(404));
+});
+
+app.use(function (err, req, res, next) {
+  console.log(err?.message)
+  res.status(err.status || 500).json({ err })
+});
+
+module.exports = app;

exporters/kafka-message-exporter/bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('kafka-message-exporter:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}